add first measurement and evaluation pipelines

config/config_injection.yaml

@@ -8,16 +8,16 @@
 # n_objects: 500
 rotate_objs: NO
 use_mpi: YES
-run_name: "test_20230509"
+run_name: "test_20230517"
 
 pos_sampling:
   # type: "HexGrid"
   # type: "RectGrid"
   # grid_spacing: 18.5 # arcsec (~250 pixels)
   type: "uniform"
-  object_density: 50 # arcmin^-2
+  object_density: 37 # arcmin^-2
 
-output_img_dir: "./workspace"
+output_img_dir: "/share/home/fangyuedong/injection_pipeline/workspace"
 input_img_list: "/share/home/fangyuedong/injection_pipeline/input_L1_img_MSC_0000000.list"
 
 ###############################################
@@ -80,3 +80,17 @@ ins_effects:
 ###############################################
 random_seeds:
   seed_Av:              121212    # Seed for generating random intrinsic extinction
+
+###############################################
+# Measurement setting
+###############################################
+measurement_setting:
+  input_img_list: "/share/home/fangyuedong/injection_pipeline/injected_L1_img_MSC_0000000.list"
+  # input_img_list: "/share/home/fangyuedong/injection_pipeline/input_L1_img_MSC_0000000.list"
+  input_wht_list: "/share/home/fangyuedong/injection_pipeline/L1_wht_img_MSC_0000000.list"
+  input_flg_list: "/share/home/fangyuedong/injection_pipeline/L1_flg_img_MSC_0000000.list"
+  input_psf_list: "/share/home/fangyuedong/injection_pipeline/psf_img_MSC_0000000.list"
+  sex_config: "/share/home/fangyuedong/injection_pipeline/config/default.config"
+  sex_param: "/share/home/fangyuedong/injection_pipeline/config/default.param"
+  n_jobs: 8
+  output_dir: "/share/home/fangyuedong/injection_pipeline/workspace"
\ No newline at end of file

evaluation/aperture_noise.py

+import argparse
+import numpy as np
+import os
+from astropy.io import fits
+from astropy.stats import sigma_clip
+from astropy.stats import median_absolute_deviation
+from astropy.stats import mad_std
+
+def get_all_stats(values_arrays):
+    """
+    """
+    stats = {}
+
+    stats['mean']   = np.mean(values_arrays)
+    stats['median'] = np.median(values_arrays)
+    stats['std']    = np.std(values_arrays)
+    stats['mad']    = mad_std(values_arrays)
+
+    return stats
+
+def define_options():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--data_image', dest='data_image', type=str, required=True,
+                        help='Name of the data image: (default: "%(default)s"')
+    parser.add_argument('--seg_image', dest='seg_image', type=str, required=True,
+                        help='Name of the mask / segmentation image: (default: "%(default)s"')
+    parser.add_argument('--flag_image', dest='flag_image', type=str, required=False,
+                        default=None, help='Name of the flag image (default: "%(default)s"')
+    parser.add_argument('--sky_image', dest='sky_image', type=str, required=False,
+                        default=None, help='Name of the sky image (default: "%(default)s"')
+    parser.add_argument('--aper_min', dest='aper_min', type=int, required=False,
+                        default=5, help='Minimum no. of pixels at level: (default: "%(default)s"')
+    parser.add_argument('--aper_max', dest='aper_max', type=int, required=False,
+                        default=20, help='Maximum no. of pixels at level: (default: "%(default)s"')
+    parser.add_argument('--aper_sampling', dest='aper_sampling', type=int, required=False,
+                        default=1, help='Minimum no. of pixels at level: (default: "%(default)s"')
+    parser.add_argument('--n_sample', dest='n_sample', type=int, required=False,
+                        default=500, help='Minimum no. of pixels at level: (default: "%(default)s"')
+    parser.add_argument('--out_basename', dest='out_basename', type=str, required=False,
+                        default="aper", help='Base name for the output names: (default: "%(default)s"')
+    parser.add_argument('--output_dir', dest='output_dir', type=str, required=False,
+                        default="./workspace", help='dir path for the output : (default: "%(default)s"')
+    return parser
+
+def create_circular_mask(h, w, center=None, radius=None):
+    if center is None: # use the middle of the image
+        center = [int(w / 2), int(h / 2)]
+    if radius is None: # use the smallest distance between the center and image walls
+        radius = min(center[0], center[1], w - center[0], h - center[1])
+
+    Y, X = np.ogrid[:h, :w]
+    dist_from_center = np.sqrt((X - center[0]) ** 2 + (Y - center[1]) ** 2)
+
+    mask = dist_from_center <= radius
+    return mask
+
+def sampling(image_data, seg_data, flag_data, aperture, Nsample):
+    wx, wy = np.where( (image_data != 0) & (seg_data == 0) & (flag_data == 0))
+    Nx, Ny = image_data.shape
+
+    flux_average = np.zeros(Nsample)
+    flux_median = np.zeros(Nsample)
+    x_position = np.zeros(Nsample)
+    y_position = np.zeros(Nsample)
+    i = 0
+    i_iter = 0
+    while i < Nsample:
+        if i_iter > 100*Nsample:
+            print('# Not enough background pixels for image depth analysis!')
+            break
+        i_iter += 1
+
+        idx = np.random.randint(len(wx))
+        stmpsize = aperture+1
+
+        if wx[idx]+stmpsize >= Nx:
+            continue
+        if wy[idx]+stmpsize >= Ny:
+            continue
+
+        img_stmp  = image_data[ wx[idx]:wx[idx]+stmpsize, wy[idx]:wy[idx]+stmpsize ]
+        seg_stmp  = seg_data[ wx[idx]:wx[idx]+stmpsize, wy[idx]:wy[idx]+stmpsize ]
+        flag_stmp = flag_data[ wx[idx]:wx[idx]+stmpsize, wy[idx]:wy[idx]+stmpsize ]
+
+        mask = create_circular_mask(stmpsize, stmpsize, center=[stmpsize//2,stmpsize//2], radius=aperture//2)
+        area = np.pi*(aperture/2)**2
+        area_sum = len(mask[mask==True])
+        ratio = area/area_sum
+
+        ss = np.sum(seg_stmp[mask])
+        if ss != 0:
+            continue
+        fs = np.sum(flag_stmp[mask])
+        if fs != 0:
+            continue
+        flux_average[i] = np.average(img_stmp[mask])
+        flux_median[i]  = np.median(img_stmp[mask])
+        x_position[i] = (wx[idx]+wx[idx]+stmpsize)/2.0
+        y_position[i] = (wy[idx]+wy[idx]+stmpsize)/2.0 
+
+        i += 1
+
+    print('Needed %i tries for %i samples!'%(i_iter, Nsample))
+
+    return flux_average, flux_median, x_position, y_position
+    
+def noise_statistics_aperture(fitsname, segname, flagname=None, sky_image=None, aperture_min=1, aperture_max=10, aperture_step=1, seed=None, Nsample=100, sigma_cl=10., base_name="aper", output_dir='./'):
+    f     = fits.open(fitsname)
+    fseg  = fits.open(segname)
+    # image_data = f[1].data
+    image_data = f[1].data * f[1].header["GAIN1"]
+    seg_data = fseg[0].data
+
+    f.close()
+    fseg.close()
+
+    if flagname:
+        fflag = fits.open(flagname)
+        flag_data = fflag[1].data
+        fflag.close()
+    else:
+        flag_data = np.zeros(image_data.shape)
+
+    if sky_image:
+        hdu = fits.open(sky_image)
+        sky_data = hdu[0].data * hdu[0].header["GAIN1"]
+        image_data -= sky_data
+
+    if seed != None:
+        np.random.seed(seed)
+    
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+
+    im = image_data
+    im[seg_data > 0] = 0.
+    hist_data = im[im != 0.].flatten()
+
+    aperture_list = np.arange(aperture_min, aperture_max+1, aperture_step, dtype=int)
+    sigma_output = np.zeros(len(aperture_list))
+    mad_output   = np.zeros(len(aperture_list))
+    mad_std_output = np.zeros(len(aperture_list))
+
+    for j, aperture in enumerate(aperture_list):
+        flux_average, flux_median, x_position, y_position = sampling(image_data, seg_data, flag_data, aperture, Nsample)
+        
+        mean_stats   = get_all_stats(flux_average)
+        median_stats = get_all_stats(flux_median)
+        print("Mean:   %e += %e +- %e"%(mean_stats['median'], mean_stats['mad'], mean_stats['std']))
+        print("Median: %e += %e +- %e"%(median_stats['median'], median_stats['mad'], median_stats['std']))
+
+        aper_file = '%s_%03i.txt'%(base_name, aperture)
+        aper_file = os.path.join(output_dir, aper_file)
+        print('Aperture file: %s'%aper_file)
+        with open(aper_file, "w+") as aper_out:
+            for one_value in zip(flux_average, flux_median, x_position, y_position):
+                one_line = "{:.7f} {:.7f} {:.1f} {:.1f}\n".format(*one_value)
+                aper_out.write(one_line)
+
+    return aperture_list, sigma_output, mad_output, mad_std_output
+
+if __name__ == "__main__":
+    args = define_options().parse_args()
+    aperture_ap, sigma_ap, mad_ap, nmad_ap = noise_statistics_aperture(
+        args.data_image,
+        args.seg_image,
+        args.flag_image,
+        args.sky_image,
+        aperture_min=args.aper_min,
+        aperture_max=args.aper_max,
+        aperture_step=args.aper_sampling,
+        Nsample=args.n_sample,
+        base_name=args.out_basename,
+        output_dir=args.output_dir)
\ No newline at end of file

evaluation/calculate_completeness_fraction.py

+import argparse
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+from astropy.io import ascii, fits
+from cross_match_catalogs import read_catalog, match_catalogs_img
+from ObservationSim.Instrument import Telescope, Filter, FilterParam
+
+VC_A = 2.99792458e+18  # speed of light: A/s
+
+def define_options():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--TU_catalog', dest='TU_catalog', type=str, required=True,
+                        help='path to the (injected) truth catalog')
+    parser.add_argument('--source_catalog', dest='source_catalog', type=str, required=True,
+                        help='path to the (extracted) injected catalog')
+    parser.add_argument('--orig_catalog', dest='orig_catalog', type=str, required=True,
+                        help='path to the (extracted) original catalog')
+    parser.add_argument('--image', dest='image', type=str, required=True,
+                        help='path to the image, used to get the header info')
+    parser.add_argument('--output_dir', dest='output_dir', type=str, required=False,
+                        default='./workspace', help='output path')
+    return parser
+
+def getChipFilter(chipID, filter_layout=None):
+        """Return the filter index and type for a given chip #(chipID)
+        """
+        filter_type_list = ["nuv","u", "g", "r", "i","z","y","GU", "GV", "GI", "FGS"]
+        if filter_layout is not None:
+            return filter_layout[chipID][0], filter_layout[chipID][1]
+
+        # updated configurations
+        if chipID>42 or chipID<1: raise ValueError("!!! Chip ID: [1,42]")
+        if chipID in [6, 15, 16, 25]:  filter_type = "y"
+        if chipID in [11, 20]:         filter_type = "z"
+        if chipID in [7, 24]:          filter_type = "i"
+        if chipID in [14, 17]:         filter_type = "u"
+        if chipID in [9, 22]:          filter_type = "r"
+        if chipID in [12, 13, 18, 19]: filter_type = "nuv"
+        if chipID in [8, 23]:          filter_type = "g"
+        if chipID in [1, 10, 21, 30]:  filter_type = "GI"
+        if chipID in [2, 5, 26, 29]:   filter_type = "GV"
+        if chipID in [3, 4, 27, 28]:   filter_type = "GU"
+        if chipID in range(31, 43):    filter_type = 'FGS'
+        filter_id = filter_type_list.index(filter_type)
+
+        return filter_id, filter_type
+
+def magToFlux(mag):
+    """
+    flux of a given AB magnitude
+
+    Parameters:
+    mag: magnitude in unit of AB
+
+    Return:
+    flux: flux in unit of erg/s/cm^2/Hz
+    """
+    flux = 10**(-0.4*(mag+48.6))
+    return flux
+
+def getElectronFluxFilt(mag, filt, tel, exptime=150.):
+    photonEnergy = filt.getPhotonE()
+    flux = magToFlux(mag)
+    factor = 1.0e4 * flux/photonEnergy * VC_A * (1.0/filt.blue_limit - 1.0/filt.red_limit)
+    return factor * filt.efficiency * tel.pupil_area * exptime
+
+def convert_catalog(catname):
+    data_dir = os.path.dirname(catname)
+    base_name = os.path.basename(catname)
+    text_file = ascii.read(catname)
+    fits_filename = os.path.join(data_dir, base_name + '.fits')
+    text_file.write(fits_filename, overwrite=True)
+
+def validation_hist(val, idx, name="val", nbins=10, fig_name='detected_counts.png', output_dir='./'):
+    counts, bins = np.histogram(val, bins=nbins)
+    is_empty = np.full(len(val), False)
+    for i in range(len(idx)):
+        if idx[i].size == 0:
+            is_empty[i] = True
+    counts_detected, _ = np.histogram(val[~is_empty], bins=nbins)
+    plt.figure()
+    plt.stairs(counts, bins, color='r', label='TU objects')
+    plt.stairs(counts_detected, bins, color='g', label='Detected')
+    plt.xlabel(name, size='x-large')
+    plt.title("Counts")
+    plt.legend(loc='upper right', fancybox=True)
+    fig_name = os.path.join(output_dir, fig_name)
+    plt.savefig(fig_name)
+    return counts, bins
+
+def hist_fraction(val, idx, name='val', nbins=10, normed=False, output_dir='./'):
+    counts, bins = np.histogram(val, bins=nbins)
+    is_empty = np.full(len(val), False)
+    for i in range(len(idx)):
+        if idx[i].size == 0:
+            is_empty[i] = True
+    counts_detected, _ = np.histogram(val[~is_empty], bins=nbins, density=normed)
+    fraction = counts_detected / counts
+    fraction[np.where(np.isnan(fraction))[0]] = 0.
+    plt.figure()
+    plt.stairs(fraction, bins, color='r', label='completeness fraction')
+    plt.xlabel(name, size='x-large')
+    plt.title("Completeness Fraction")
+    fig_name = os.path.join(output_dir, "completeness_fraction_%s.png"%(name))
+    plt.savefig(fig_name)
+    return fraction
+
+def calculate_fraction(TU_catalog, source_catalog, output_dir, nbins=10):
+    convert_catalog(TU_catalog)
+    x_TU, y_TU, col_list = read_catalog(TU_catalog + '.fits', ext_num=1, ra_name="xImage", dec_name="yImage", col_list=["mag"])
+    mag_TU = col_list[0]
+    x_source, y_source, _ = read_catalog(source_catalog, ext_num=1, ra_name="X_IMAGE", dec_name="Y_IMAGE")
+    idx1, idx2, = match_catalogs_img(x1=x_TU, y1=y_TU, x2=x_source, y2=y_source)
+    counts, bins = validation_hist(val=mag_TU, idx=idx1, name="mag_injected", output_dir=output_dir)
+    fraction = hist_fraction(val=mag_TU, idx=idx1, name="mag_injected", nbins=10, output_dir=output_dir)
+    return counts, bins, fraction
+
+def calculate_undetected_flux(orig_cat, mag_bins, fraction, mag_low=20.0, mag_high=26.0, image=None,  output_dir='./'):
+    # Get info from original image
+    hdu = fits.open(image)
+    header0 = hdu[0].header
+    header1 = hdu[1].header
+    nx_pix, ny_pix = header0["PIXSIZE1"], header0["PIXSIZE2"]
+    exp_time = header0["EXPTIME"]
+    gain = header1["GAIN1"]
+    chipID = int(header0["DETECTOR"][-2:])
+    zp = header1["ZP"]
+    hdu.close()
+
+    # Get info from original catalog
+    ra_orig, dec_orig, col_list_orig = read_catalog(orig_cat, ra_name='RA', dec_name='DEC', col_list=['Mag_Kron'])
+    mag_orig = col_list_orig[0]
+    nbins = len(mag_bins) - 1
+    counts, _ = np.histogram(mag_orig, bins=nbins)
+    
+    mags = (mag_bins[:-1] + mag_bins[1:])/2.
+    counts_missing = (counts / fraction) - counts
+    counts_missing[np.where(np.isnan(counts_missing))[0]] = 0.
+    counts_missing[np.where(np.isinf(counts_missing))[0]] = 0.
+    print(counts_missing)
+    print(counts_missing.sum())
+
+    plt.figure()
+    plt.stairs(counts_missing, mag_bins, color='r', label='undetected counts')
+    plt.xlabel("mag_injected", size='x-large')
+    plt.title("Undetected Sources")
+    fig_name = os.path.join(output_dir, "undetected_sources.png")
+    plt.savefig(fig_name)
+
+    tel = Telescope()
+    filter_param = FilterParam()
+    filter_id, filter_type = getChipFilter(chipID=chipID)
+
+    filt = Filter(filter_id=filter_id,
+                filter_type=filter_type,
+                filter_param=filter_param)
+    
+    undetected_flux = 0.
+    for i in range(len(mags)):
+        if mags[i] < mag_low or mags[i] > mag_high:
+            continue
+        flux_electrons = counts_missing[i] * getElectronFluxFilt(mag=mags[i], filt=filt, tel=tel)
+        undetected_flux += flux_electrons
+
+    undetected_flux /= (float(nx_pix) * float(ny_pix))
+    return undetected_flux
+
+if __name__ == "__main__":
+    args = define_options().parse_args()
+    counts, bins, fraction = calculate_fraction(
+        TU_catalog=args.TU_catalog,
+        source_catalog=args.source_catalog,
+        output_dir=args.output_dir,
+        nbins=20
+    )
+    undetected_flux = calculate_undetected_flux(
+        orig_cat=args.orig_catalog,
+        mag_bins=bins,
+        fraction=fraction,
+        image=args.image,
+        output_dir=args.output_dir,
+    )
+    print(undetected_flux)
\ No newline at end of file

evaluation/cross_match_catalogs.py

@@ -41,8 +41,8 @@ def match_catalogs_sky(ra1, dec1, ra2, dec2, max_dist=0.6, others1=[], others2=[
 def match_catalogs_img(x1, y1, x2, y2, max_dist=0.5, others1=[], others2=[], thresh=[]):
     cat1 = np.array([(x, y) for x,y in zip(x1, y1)])
     cat2 = np.array([(x, y) for x,y in zip(x2, y2)])
-    print(np.shape(cat1))
-    print(np.shape(cat2))
+    # print(np.shape(cat1))
+    # print(np.shape(cat2))
     tree = BallTree(cat2)
     idx1 = tree.query_radius(cat1, r = max_dist)
     tree = BallTree(cat1)
@@ -59,7 +59,7 @@ def match_catalogs_img(x1, y1, x2, y2, max_dist=0.5, others1=[], others2=[], thr
 
 def validation_hist(val1, idx1, name="val1", nbins=10):
     counts, bins = np.histogram(val1)
-    plt.stairs(bins, counts)
+    plt.stairs(counts, bins)
     plt.xlabel(name, size='x-large')
     plt.savefig("detection_completeness.png")
     # plt.show()

injection_pipeline/Catalog/C5_SimCat.py

@@ -69,7 +69,7 @@ class SimCat(CatalogBase):
 
         for i in range(ngals):
             param = self.initialize_param()
-            igals = ngals % max_ngals
+            igals = i % max_ngals
             param['ra'] = gals['ra_true'][igals]
             param['dec'] = gals['dec_true'][igals]
 

injection_pipeline/SingleEpochImage.py

@@ -28,6 +28,7 @@ class SingleEpochImage(object):
         elif 'object_density' in self.config["pos_sampling"] and self.config["pos_sampling"]['object_density'] is not None:
             # Fixed number density of objects
             self.objs_per_real = round(self.u_area * self.config["pos_sampling"]['object_density'])
+            print("number of injected obj = ",self.objs_per_real)
         else:
             # Grid types: calculate nobjects later
             self.objs_per_real = None
@@ -56,22 +57,24 @@ class SingleEpochImage(object):
         
         self.setup_image_for_injection()
 
-        self.header_ext = generateExtensionHeader(
-            chip=self.chip,
-            xlen=self.chip.npix_x,
-            ylen=self.chip.npix_y,
-            ra=self.ra_cen,
-            dec=self.dec_cen,
-            pa=self.pos_ang,
-            gain=self.chip.gain, # {TODO}
-            readout=self.chip.read_noise,
-            dark=self.chip.dark_noise,
-            saturation=90000,
-            pixel_scale=self.chip.pix_scale, 
-            pixel_size=self.chip.pix_size,
-            xcen=self.chip.x_cen,
-            ycen=self.chip.y_cen,
-            extName = 'raw')
+        # [TODO] Header should come from input image
+        # self.header_ext = generateExtensionHeader(
+        #     chip=self.chip,
+        #     xlen=self.chip.npix_x,
+        #     ylen=self.chip.npix_y,
+        #     ra=self.ra_cen,
+        #     dec=self.dec_cen,
+        #     pa=self.pos_ang,
+        #     gain=self.chip.gain, # {TODO}
+        #     readout=self.chip.read_noise,
+        #     dark=self.chip.dark_noise,
+        #     saturation=90000,
+        #     pixel_scale=self.chip.pix_scale, 
+        #     pixel_size=self.chip.pix_size,
+        #     xcen=self.chip.x_cen,
+        #     ycen=self.chip.y_cen,
+        #     extName = 'raw')
+        self.header_ext = self.header_img
 
         # (TODO) specify sub-directory
         self.chip_output = ChipOutput(
@@ -100,19 +103,22 @@ class SingleEpochImage(object):
     def read_initial_image(self, filepath):
         data_dir = os.path.dirname(filepath)
         img_name = os.path.basename(filepath)
-        data = fits.open(filepath)
-        self.header0 = data[0].header
-        self.header_img = data[1].header
+        hdu = fits.open(filepath)
+        self.header0 = hdu[0].header
+        self.header_img = hdu[1].header
         self.image = fits.getdata(filepath)
+        hdu.close()
         
         # Determine which CCD
         self.chip_ID = int(self.header0['DETECTOR'][-2:])
         # Construnct Chip object
         self.chip = Chip(chipID=self.chip_ID, config=self.config)
         self.exp_time = float(self.header0 ['EXPTIME'])
+        self.chip.gain = float(self.header_img["GAIN1"])
 
         # Process L1 image
-        self.image = self.image * self.exp_time / self.chip.gain
+        # self.image = self.image * self.exp_time / self.chip.gain
+        self.image = self.image * self.chip.gain
 
         # Process L1 SKY image
         # [TODO]
@@ -173,6 +179,8 @@ class SingleEpochImage(object):
         # Make sure we have enough objects to inject
         assert nobj <= len(cat.objs)
 
+        chip_wcs = galsim.FitsWCS(header=self.header_ext)
+        
         for i in range(nobj):
             obj = cat.objs[i]
             try:
@@ -190,7 +198,7 @@ class SingleEpochImage(object):
             # Update object position to a point on grid
             obj.param['ra'], obj.param['dec'] = pos[i][0], pos[i][1]
             self.chip_output.Log_info("ra = %.3f, dec = %.3f"%(obj.param['ra'], obj.param['dec']))
-            pos_img, offset, local_wcs, real_wcs, fd_shear = obj.getPosImg_Offset_WCS(img=self.chip.img, fdmodel=self.fd_model, chip=self.chip, verbose=False, img_header=self.header_ext)
+            pos_img, offset, local_wcs, real_wcs, fd_shear = obj.getPosImg_Offset_WCS(img=self.chip.img, fdmodel=self.fd_model, chip=self.chip, verbose=False, chip_wcs=chip_wcs)
             self.chip_output.Log_info("pos_img_x = %.3f, pos_img_y = %.3f"%(pos_img.x, pos_img.y))
             try:
                 isUpdated, pos_shear = obj.drawObj_multiband(
@@ -225,8 +233,9 @@ class SingleEpochImage(object):
         self.chip.img = galsim.Image(self.chip.img.array, dtype=np.float32)
         
         # [TODO] TEST
-        self.chip.img *= self.gain
-        self.chip.img /= self.exp_time
+        # self.chip.img *= self.chip.gain
+        # self.chip.img /= self.exp_time
+        self.chip.img /= self.chip.gain
 
         hdu1 = fits.PrimaryHDU(header=self.header0)
         hdu2 = fits.ImageHDU(self.chip.img.array, header=self.header_img)

injection_pipeline/injection_pipeline.py

@@ -33,11 +33,17 @@ class InjectionPipeline(object):
 
         # Prepare the output directory
         if not os.path.exists(self.config["output_img_dir"]):
+            try:
                 os.makedirs(self.config["output_img_dir"])
+            except OSError:
+                pass
         
         self.output_dir = os.path.join(self.config["output_img_dir"], self.config["run_name"])
         if not os.path.exists(self.output_dir):
+            try:
                 os.makedirs(self.output_dir)
+            except OSError:
+                pass
 
     def run_injection_img_list(self, Catalog):
         if self.config["use_mpi"]:

measurement_pipeline/create_mask.py

+import numpy as np
+import argparse
+import os
+from skimage.draw import ellipse
+from astropy.io import fits
+
+def define_options():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--catalog', dest='catalog_path', type=str, required=False,
+                        default=None, help='path to the object catalog (default: "%(default)s"')
+    parser.add_argument('--flag_image', dest='flag_path', type=str, required=False,
+                        default=None, help='path to the flag image (default: "%(default)s"')
+    parser.add_argument('--output_dir', dest='output_dir', type=str,
+                        default=None, help='path to the output mask image (default: "%(default)s"')
+    parser.add_argument('--margin', dest='margin', type=float,
+                        default=0, help='margin for major and minor axes')
+    return parser
+
+def mask_object(img, x_image, y_image, A_image, B_image, theta_image, margin=0):
+    rr, cc = ellipse(
+        r=y_image,
+        c=x_image,
+        r_radius=B_image,
+        c_radius=A_image,
+        shape=img.shape,
+        rotation=np.deg2rad(-theta_image)
+    )
+    img[rr, cc] = 1
+
+def create_mask_catalog(catalog_path, flag_path, output_dir, margin=0):
+    catalog_dir = os.path.dirname(catalog_path)
+    catalog_name = os.path.basename(catalog_path)
+    flag_dir = os.path.dirname(flag_path)
+    flag_name = os.path.basename(flag_path)
+
+    # Read header and data from flag image
+    hdu = fits.open(flag_path)
+    header0 = hdu[0].header
+    header1 = hdu[1].header
+    flag_img = fits.getdata(flag_path)
+    hdu.close()
+
+    # Read data from detection catalog
+    hdu = fits.open(catalog_path)
+    catalog = hdu[1].data
+    hdu.close()
+
+    # Setup mask image
+    mask = flag_img.copy()
+    mask[mask != 0] = 1
+
+    # Columns used for mask creation
+    x_image = catalog["X_IMAGE"]
+    y_image = catalog["Y_IMAGE"]
+    A_image = catalog["A_IMAGE"]
+    B_image = catalog["B_IMAGE"]
+    theta_image = catalog["theta_image"]
+
+    for i in range(len(catalog)):
+        mask_object(
+            img=mask,
+            x_image=x_image[i],
+            y_image=y_image[i],
+            A_image=A_image[i],
+            B_image=B_image[i],
+            theta_image=theta_image[i],
+            margin=margin
+        )
+    
+    # Save flag image
+    fname = os.path.join(output_dir, flag_name.replace("flg_L1", "mask_L1"))
+    hdu1 = fits.PrimaryHDU(header=header0)
+    hdu2 = fits.ImageHDU(mask, header=header1)
+    hdu1 = fits.HDUList([hdu1, hdu2])
+    hdu1.writeto(fname, output_verify='ignore', overwrite=True)
+
+if __name__ == "__main__":
+    args = define_options().parse_args()
+    create_mask_catalog(
+        catalog_path=args.catalog_path,
+        flag_path=args.flag_path,
+        output_dir=args.output_dir,
+        margin=args.margin
+    )
\ No newline at end of file

measurement_pipeline/data/default.config

+# Default configuration file for SExtractor 2.25.0
+# EB 2019-10-03
+#
+ 
+#-------------------------------- Catalog ------------------------------------
+ 
+CATALOG_NAME     test.cat       # name of the output catalog
+CATALOG_TYPE     FITS_1.0     # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT,
+                                # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC
+PARAMETERS_NAME  default.param  # name of the file containing catalog contents
+ 
+#------------------------------- Extraction ----------------------------------
+ 
+DETECT_TYPE      CCD            # CCD (linear) or PHOTO (with gamma correction)
+DETECT_MINAREA   3              # min. # of pixels above threshold
+DETECT_THRESH    1.5            # <sigmas> or <threshold>,<ZP> in mag.arcsec-2
+ANALYSIS_THRESH  1.5            # <sigmas> or <threshold>,<ZP> in mag.arcsec-2
+THRESH_TYPE      RELATIVE
+ 
+FILTER           Y              # apply filter for detection (Y or N)?
+FILTER_NAME      ../measurement_pipeline/data/default.conv   # name of the file containing the filter
+ 
+DEBLEND_NTHRESH  64             # Number of deblending sub-thresholds
+DEBLEND_MINCONT  0.0005          # Minimum contrast parameter for deblending
+ 
+CLEAN            Y              # Clean spurious detections? (Y or N)?
+CLEAN_PARAM      1.0            # Cleaning efficiency
+ 
+MASK_TYPE        CORRECT        # type of detection MASKing: can be one of
+                                # NONE, BLANK or CORRECT
+
+#------------------------------ Flagging -----------------------------------
+#FLAG_TYPE        OR
+
+#------------------------------ Photometry -----------------------------------
+ 
+PHOT_APERTURES   5              # MAG_APER aperture diameter(s) in pixels
+PHOT_AUTOPARAMS  2.5, 3.5       # MAG_AUTO parameters: <Kron_fact>,<min_radius>
+PHOT_PETROPARAMS 2.0, 3.5       # MAG_PETRO parameters: <Petrosian_fact>,
+                                # <min_radius>
+ 
+SATUR_LEVEL      50000.0        # level (in ADUs) at which arises saturation
+SATUR_KEY        SATURATE       # keyword for saturation level (in ADUs)
+ 
+MAG_ZEROPOINT    0.0            # magnitude zero-point
+#MAG_GAMMA        4.0            # gamma of emulsion (for photographic scans)
+GAIN             1.0            # detector gain in e-/ADU
+#GAIN_KEY         GAIN           # keyword for detector gain in e-/ADU
+PIXEL_SCALE      0            # size of pixel in arcsec (0=use FITS WCS info)
+ 
+#------------------------- Star/Galaxy Separation ----------------------------
+ 
+SEEING_FWHM      0.15           # stellar FWHM in arcsec
+STARNNW_NAME     ../measurement_pipeline/data/default.nnw    # Neural-Network_Weight table filename
+ 
+#------------------------------ Background -----------------------------------
+
+BACK_TYPE        AUTO
+BACK_SIZE        400            # Background mesh: <size> or <width>,<height>
+BACK_FILTERSIZE  3              # Background filter: <size> or <width>,<height>
+ 
+BACKPHOTO_TYPE   GLOBAL         # can be GLOBAL or LOCAL
+ 
+#------------------------------ Check Image ----------------------------------
+ 
+CHECKIMAGE_TYPE  NONE           # can be NONE, BACKGROUND, BACKGROUND_RMS,
+                                # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND,
+                                # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION,
+                                # or APERTURES
+CHECKIMAGE_NAME  ../check.fits  # Filename for the check-image
+ 
+#--------------------- Memory (change with caution!) -------------------------
+ 
+MEMORY_OBJSTACK  3000           # number of objects in stack
+MEMORY_PIXSTACK  300000         # number of pixels in stack
+MEMORY_BUFSIZE   1024           # number of lines in buffer
+ 
+#----------------------------- Miscellaneous ---------------------------------
+ 
+VERBOSE_TYPE     NORMAL         # can be QUIET, NORMAL or FULL
+HEADER_SUFFIX    .head          # Filename extension for additional headers
+WRITE_XML        N              # Write XML file (Y/N)?
+XML_NAME         sex.xml        # Filename for XML output
+
+#------------------------------ Weighting -----------------------------------
+#WEIGHT_TYPE     MAP_WEIGHT
+#RESCALE_WEIGHTS Y
+#WEIGHT_IMAGE    weight.fits
+#WEIGHT_GAIN     N
+#WEIGHT_THRESH   1e-7

measurement_pipeline/data/default.param

+# ID and flags
+NUMBER 
+FLAGS 
+# Basic coordinates
+X_IMAGE
+Y_IMAGE
+XMIN_IMAGE 
+XMAX_IMAGE 
+YMIN_IMAGE 
+YMAX_IMAGE 
+# Accurate coordinates
+XWIN_IMAGE 
+YWIN_IMAGE 
+ERRAWIN_IMAGE
+ERRBWIN_IMAGE
+ERRTHETAWIN_IMAGE
+ALPHAWIN_J2000 
+DELTAWIN_J2000 
+ERRAWIN_WORLD
+ERRBWIN_WORLD
+ERRTHETAWIN_J2000
+# Peak value
+XPEAK_IMAGE 
+YPEAK_IMAGE 
+ALPHAPEAK_J2000 
+DELTAPEAK_J2000 
+# Basic shape info
+A_IMAGE
+B_IMAGE
+THETA_IMAGE
+A_WORLD
+B_WORLD
+THETA_J2000
+ELONGATION
+FLUX_RADIUS
+# Isophotal photometry
+#FLUX_ISO
+#FLUXERR_ISO
+MAG_ISO
+MAGERR_ISO
+ISOAREA_IMAGE
+ISOAREAF_IMAGE
+ISOAREA_WORLD
+ISOAREAF_WORLD
+# Aperture photometry
+FLUX_APER(1)
+FLUXERR_APER(1)
+MAG_APER(1)
+MAGERR_APER(1)
+# Kron photometry
+FLUX_AUTO
+FLUXERR_AUTO
+MAG_AUTO
+MAGERR_AUTO
+KRON_RADIUS
+# Petrosian photometry
+#FLUX_PETRO
+#FLUXERR_PETRO
+MAG_PETRO
+MAGERR_PETRO
+PETRO_RADIUS
+# Basic surface brightnesses
+BACKGROUND
+THRESHOLD
+MU_THRESHOLD
+FLUX_MAX
+MU_MAX
\ No newline at end of file

measurement_pipeline/detection.py

+import os
+import subprocess
+import joblib
+from subprocess import Popen
+
+# try:
+#     import importlib.resources as pkg_resources
+# except ImportError:
+#     # Try backported to PY<37 'importlib_resources'
+#     import importlib_resources as pkg_resources
+
+class Detection(object):
+    def __init__(self, n_jobs=4, output_dir=None):
+        self.CONFIG_PATH = os.path.join(os.path.dirname(__file__), "data/")
+        self.n_jobs = n_jobs
+        if output_dir:
+            self.output_dir = output_dir
+        else:
+            self.output_dir = os.path.dirname(__file__)
+    
+    def run_sextractor(self, image, weight, flag, psf=None, sex_config=None, sex_param=None, output_cat=None):
+        if output_cat is None:
+            basename = os.path.basename(image)
+            output_cat = basename.replace(".fits", "_extracted.cat")
+            output_cat = os.path.join(self.output_dir, output_cat)
+        
+        if sex_config is None:
+            sex_config = os.path.join(self.CONFIG_PATH, "default.config")
+        if sex_param is None:
+            sex_param = os.path.join(self.CONFIG_PATH, "default.param")
+        filter_path = os.path.join(self.CONFIG_PATH, "default.conv")
+        nnw_path = os.path.join(self.CONFIG_PATH, "default.nnw")
+        
+        sex_cmd1 = 'sex ' + image + ', ' + image
+        sex_cmd2 = ' -c ' + sex_config
+        sex_cmd3 = ' -CATALOG_NAME ' + output_cat + ' -WEIGHT_IMAGE ' + weight \
+                      + ' -FLAG_IMAGE ' + flag
+        sex_cmd4 = ' -PARAMETERS_NAME ' + sex_param + ' -FILTER_NAME ' + filter_path \
+                      + ' -STARNNW_NAME ' + nnw_path
+        sex_cmd5 = ''
+        if psf is not None:
+            sex_cmd5 += ' -PSF_NAME ' + psf
+        sex_cmd = sex_cmd1 + sex_cmd2 + sex_cmd3 + sex_cmd4 + sex_cmd5
+        print(sex_cmd)
+        p = Popen(sex_cmd, shell=True)
+        p.wait()
+
+    def run_detection(self, image_list, weight_list, flag_list, psf_list=None, sex_config=None, sex_param=None, output_cat_list=None):
+        assert len(image_list) == len(weight_list), "length of image list must be equal to length of weight list"
+        assert len(image_list) == len(flag_list), "length of image list must be equal to length of flag list"
+        if psf_list:
+            assert len(image_list) == len(psf_list), "length of image list must be equal to length of psf list"
+        else:
+            psf_list = [None for i in range(len(image_list))]
+        if output_cat_list:
+            assert len(image_list) == len(output_cat_list), "length of image list must be equal to length of output catalog path list"
+        else:
+            output_cat_list = [None for i in range(len(image_list))]
+        
+        joblib.Parallel(n_jobs=self.n_jobs)(joblib.delayed(self.run_sextractor)(image=image_list[i], weight=weight_list[i], flag=flag_list[i], psf=psf_list[i], sex_config=sex_config, sex_param=sex_param, output_cat=output_cat_list[i]) for i in range(len(image_list)))
+            
+        
\ No newline at end of file

measurement_pipeline/run_measurement.py

+import argparse
+import os
+import yaml
+from detection import Detection
+
+def parse_args():
+    '''
+    Parse command line arguments. Many of the following
+    can be set in the .yaml config file as well.
+    '''
+    parser = argparse.ArgumentParser()
+    parser.add_argument('config_file', help='.yaml config file for injection settings.')
+    parser.add_argument('-c', '--config_dir', help='Directory that houses the .yaml config file.')
+    # parser.add_argument('-d', '--data_dir', help='Directory that houses the input data.')
+    # parser.add_argument('-w', '--work_dir', help='The path for output.')
+    return parser.parse_args()
+
+class MeasurementPipeline(object):
+    def __init__(self):
+        # Load configuration
+        args = parse_args()
+        if args.config_dir is None:
+            args.config_dir = ''
+        args.config_dir = os.path.abspath(args.config_dir)
+        args.config_file = os.path.join(args.config_dir, args.config_file)
+        with open(args.config_file, "r") as stream:
+            try:
+                self.config = yaml.safe_load(stream)
+                for key, value in self.config.items():
+                    print (key + " : " + str(value))
+            except yaml.YAMLError as exc:
+                print(exc)
+        with open(self.config["measurement_setting"]["input_img_list"]) as input_list:
+            self.img_list = [line.rstrip() for line in input_list]
+        with open(self.config["measurement_setting"]["input_wht_list"]) as input_list:
+            self.wht_list = [line.rstrip() for line in input_list]
+        with open(self.config["measurement_setting"]["input_flg_list"]) as input_list:
+            self.flg_list = [line.rstrip() for line in input_list]
+        if self.config["measurement_setting"]["input_psf_list"]:
+            with open(self.config["measurement_setting"]["input_psf_list"]) as input_list:
+                self.psf_list = [line.rstrip() for line in input_list]
+        else:
+            self.psf_list = None
+
+    def run_pipeline(self):
+
+        # Detection
+        detection = Detection(
+            n_jobs=self.config["measurement_setting"]["n_jobs"],
+            output_dir=os.path.join(
+                self.config["measurement_setting"]["output_dir"],
+                self.config["run_name"]
+                )
+            )
+        detection.run_detection(
+            image_list=self.img_list,
+            weight_list=self.wht_list,
+            flag_list=self.flg_list,
+            psf_list=self.psf_list,
+            sex_config=self.config["measurement_setting"]["sex_config"],
+            sex_param=self.config["measurement_setting"]["sex_param"],
+        )
+
+if __name__ == "__main__":
+    pipeline = MeasurementPipeline()
+    pipeline.run_pipeline()
\ No newline at end of file

run_aperture_noise.sh

+#!/bin/bash
+
+python /share/home/fangyuedong/injection_pipeline/evaluation/aperture_noise.py \
+    --data_image /share/L1Result/C6_2sq_sheared_photometry/MSC_0000000/CSST_MSC_MS_SCI_20240817060512_20240817060742_100000000_08_img_L1.fits \
+    --seg_image /share/L1Result/C6_2sq_sheared_photometry/MSC_0000000/CSST_MSC_MS_SCI_20240817060512_20240817060742_100000000_08_seg.fits \
+    --flag_image /share/L1Result/C6_2sq_sheared_photometry/MSC_0000000/CSST_MSC_MS_SCI_20240817060512_20240817060742_100000000_08_flg_L1.fits \
+    --sky_image /share/L1Result/C6_2sq_sheared_photometry/MSC_0000000/CSST_MSC_MS_SCI_20240817060512_20240817060742_100000000_08_sky.fits \
+    --aper_min 20 \
+    --aper_max 50 \
+    --aper_sampling 5 \
+    --output_dir /share/home/fangyuedong/injection_pipeline/workspace/test_20230517
\ No newline at end of file

run_calculate_completeness_fraction.sh

+#!/bin/bash
+
+python /share/home/fangyuedong/injection_pipeline/evaluation/calculate_completeness_fraction.py \
+    --TU_catalog /share/home/fangyuedong/injection_pipeline/workspace/test_20230517/CSST_MSC_MS_SCI_20240817060512_20240817060742_100000000_08_img_L1_injected.cat \
+    --source_catalog /share/home/fangyuedong/injection_pipeline/workspace/test_20230517/CSST_MSC_MS_SCI_20240817060512_20240817060742_100000000_08_img_L1_injected_extracted.cat \
+    --orig_catalog /share/L1Result/C6_2sq_sheared_photometry/MSC_0000000/CSST_MSC_MS_SCI_20240817060512_20240817060742_100000000_08_cat.fits \
+    --image /share/L1Result/C6_2sq_sheared_photometry/MSC_0000000/CSST_MSC_MS_SCI_20240817060512_20240817060742_100000000_08_img_L1.fits \
+    --output_dir /share/home/fangyuedong/injection_pipeline/workspace/test_20230517
\ No newline at end of file

run_injection.pbs

@@ -3,7 +3,7 @@
 #PBS -N INJ
 #PBS -l nodes=wcl-1:ppn=60
 #PBS -u fangyuedong
-#PBS -j oe
+####PBS -j oe
 
 cd $PBS_O_WORKDIR
 

run_make_mask.sh

+#!/bin/bash
+
+python /share/home/fangyuedong/injection_pipeline/measurement_pipeline/create_mask.py \
+    --catalog /share/home/fangyuedong/injection_pipeline/workspace/test_20230517/CSST_MSC_MS_SCI_20240817060512_20240817060742_100000000_08_img_L1_extracted.cat \
+    --flag_image /share/L1Result/C6_2sq_sheared_photometry/MSC_0000000/CSST_MSC_MS_SCI_20240817060512_20240817060742_100000000_08_flg_L1.fits \
+    --output_dir /share/home/fangyuedong/injection_pipeline/workspace/test_20230517/ \
+    --margin 10
\ No newline at end of file

run_measurement.sh

+#!/bin/bash
+
+python /share/home/fangyuedong/injection_pipeline/measurement_pipeline/run_measurement.py config_injection.yaml -c /share/home/fangyuedong/injection_pipeline/config
\ No newline at end of file