Merge branch 'master' into 'release_v3.0'

Release version v3.2.0

See merge request !33

catalog/C9_Catalog.py

@@ -380,7 +380,7 @@ class Catalog(CatalogBase):
                 obj = Galaxy(param, logger=self.logger)
 
             # Need to deal with additional output columns for (host) galaxy
-            obj.additional_output_str = self.add_fmt % (0., 0., 0., 0., 0., 0.,
+            obj.additional_output_str = self.add_fmt % (param['mw_Av'], 0., 0., 0., 0., 0.,
                                                         param['bulgemass'], param['diskmass'], param['detA'],
                                                         param['e1'], param['e2'], param['kappa'], param['g1'], param['g2'], param['size'],
                                                         param['galType'], param['veldisp'])
@@ -469,6 +469,11 @@ class Catalog(CatalogBase):
 
             self.objs.append(obj)
 
+    def free_mem(self, **kward):
+        if not self.config["catalog_options"]["galaxy_only"]:
+            self.starDDL.freeGlobeData()
+            del self.starDDL
+
     def _load(self, **kwargs):
         self.objs = []
         self.ids = 0

catalog/testCat_fits.py

@@ -177,7 +177,7 @@ class Catalog(CatalogBase):
             # param["CSSTmag"]= True
             # param["mag_r"] = 20.
             # param['']
-            ### more keywords for stamp###
+            # more keywords for stamp#
             param['image'] = hdu[0].data
             param['image'] = param['image']/(np.sum(param['image']))
             obj = Stamp(param)

config/config_overall.yaml

@@ -12,6 +12,7 @@
 # ok to pass either way or both, as long as they are consistent
 work_dir: "/public/home/fangyuedong/project/workplace/"
 run_name: "ext_on"
+data_set: "csst-msc-c9-25sqdeg-v_test"
 
 # Project cycle and run counter are used to name the outputs
 project_cycle: 9

config/obs_config_SCI.yaml

@@ -62,10 +62,10 @@ call_sequence:
     hot_pixels: YES
     dead_pixels: YES
     bad_columns: YES
-  # Apply response nonlinearity
-  nonlinearity: {}
   # Apply CCD Saturation & Blooming
   blooming: {}
+  # Apply response nonlinearity
+  nonlinearity: {}
   # Run CTE simulation
   # CTE_effect: {}
   # Add prescan and overscan
@@ -74,6 +74,8 @@ call_sequence:
   # Add bias
   bias:
     bias_16channel: YES
+  # Add cross-talk
+  cross_talk: {}
   # Add readout noise
   readout_noise: {}
   # Apply gain

observation_sim/ObservationSim.py

@@ -25,7 +25,7 @@ class Observation(object):
 
     def prepare_chip_for_exposure(self, chip, ra_cen, dec_cen, pointing, wcs_fp=None, slsPSFOptim=False):
         # Get WCS for the focal plane
-        if wcs_fp == None:
+        if wcs_fp is None:
             wcs_fp = self.focal_plane.getTanWCS(
                 ra_cen, dec_cen, pointing.img_pa, chip.pix_scale)
 
@@ -62,6 +62,8 @@ class Observation(object):
         # Get flat, shutter, and PRNU images
         chip.flat_img, _ = chip_utils.get_flat(
             img=chip.img, seed=int(self.config["random_seeds"]["seed_flat"]))
+        if chip.chipID <= 30:
+            chip.flat_img = chip.flat_img*chip_utils.get_innerflat(chip=chip)
         if chip.chipID > 30:
             chip.shutter_img = np.ones_like(chip.img.array)
         else:
@@ -154,13 +156,12 @@ class Observation(object):
                 chip_output.Log_error(e)
                 chip_output.Log_error("Failed simulation on step: %s" % (step))
                 break
-
-        chip_output.Log_info("check running:1: pointing-%d chip-%d pid-%d memory-%6.2fGB" % (pointing.id,
-                             chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024)))
         del chip.img
         del chip.flat_img
         del chip.prnu_img
         del chip.shutter_img
+        chip_output.Log_info("check running:1: pointing-%d chip-%d pid-%d memory-%6.2fGB" % (pointing.id,
+                             chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024)))
 
     def runExposure_MPI_PointingList(self, pointing_list, chips=None):
         comm = MPI.COMM_WORLD
@@ -239,5 +240,6 @@ class Observation(object):
                     "finished running chip#%d..." % (chip.chipID))
                 for handler in chip_output.logger.handlers[:]:
                     chip_output.logger.removeHandler(handler)
+                del chip_output
                 gc.collect()
             process_counter += nchips_per_fp

observation_sim/_util.py

@@ -26,7 +26,7 @@ def parse_args():
     return parser.parse_args()
 
 
-def generate_pointing_list(config, pointing_filename=None, data_dir=None):
+def generate_pointing_list(config, pointing_filename=None, data_dir=None, dataset=''):
     pointing_list = []
 
     # Only valid when the pointing list does not contain time stamp column
@@ -63,7 +63,8 @@ def generate_pointing_list(config, pointing_filename=None, data_dir=None):
                 continue
             line = line.strip()
             columns = line.split()
-            pointing = Pointing(obs_config_file=obs_config_file)
+            pointing = Pointing(
+                obs_config_file=obs_config_file, dataset=dataset)
             pointing.read_pointing_columns(columns=columns, id=ipoint)
             t += delta_t * 60.
             pointing_list.append(pointing)
@@ -83,7 +84,8 @@ def generate_pointing_list(config, pointing_filename=None, data_dir=None):
             timestamp=t,
             exp_time=exp_time,
             pointing_type='SCI',
-            obs_config_file=obs_config_file
+            obs_config_file=obs_config_file,
+            dataset=dataset
         )
         t += delta_t * 60.
         pointing_list.append(pointing)

observation_sim/astrometry/Astrometry_util.py

@@ -12,8 +12,8 @@ def checkInputList(input_list, n):
     if not isinstance(input_list, list):
         raise TypeError("Input type is not list!", input_list)
     for i in input_list:
-        if type(i) != type(1.1):
-            if type(i) != type(1):
+        if type(i) is not type(1.1):
+            if type(i) is not type(1):
                 raise TypeError(
                     "Input list's element is not float or int!", input_list)
     if len(input_list) != n:

observation_sim/config/ChipOutput.py

@@ -28,7 +28,8 @@ class ChipOutput(object):
             sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz],
             project_cycle=self.config["project_cycle"],
             run_counter=self.config["run_counter"],
-            chip_name=self.chip_label)
+            chip_name=self.chip_label,
+            dataset=pointing.dataset)
 
         obs_id = _util.get_obs_id(img_type=self.pointing_type, project_cycle=config["project_cycle"], run_counter=config[
                                   "run_counter"], pointing_id=pointing.obs_id, pointing_type_code=pointing.pointing_type_code)

observation_sim/config/Pointing.py

@@ -10,7 +10,7 @@ import observation_sim.instruments._util as _util
 
 
 class Pointing(object):
-    def __init__(self, id=0, ra=0., dec=0., img_pa=0., timestamp=1621915200, sat_x=0., sat_y=0., sat_z=0., sun_x=0., sun_y=0., sun_z=0., sat_vx=0., sat_vy=0., sat_vz=0., exp_time=150., pointing_type='SCI', pointing_type_code='101', pointing_id='00000001', obs_config_file=None, t_shutter_open=1.3, t_shutter_close=1.3):
+    def __init__(self, id=0, ra=0., dec=0., img_pa=0., timestamp=1621915200, sat_x=0., sat_y=0., sat_z=0., sun_x=0., sun_y=0., sun_z=0., sat_vx=0., sat_vy=0., sat_vz=0., exp_time=150., pointing_type='SCI', pointing_type_code='101', pointing_id='00000001', obs_config_file=None, t_shutter_open=1.3, t_shutter_close=1.3, dataset='csst-msc-c9-25sqdeg-v3'):
         self.id = id
         self.ra = ra
         self.dec = dec
@@ -41,6 +41,7 @@ class Pointing(object):
                 self.pointing_type_code = self.obs_param["obs_type_code"]
             if self.obs_param["obs_id"]:
                 self.obs_id = str(self.obs_param["obs_id"])
+        self.dataset = dataset
 
     def get_full_depth_exptime(self, filter_type):
         if self.survey_field_type == 'WIDE':
@@ -88,7 +89,7 @@ class Pointing(object):
             self.exp_time = float(columns[18])
             is_deep = float(columns[19])
             # [TODO] Can also define other survey types
-            if is_deep != -1.0:
+            if is_deep == 2:
                 self.survey_field_type = "DEEP"
 
             if not self.obs_config_file:
@@ -114,12 +115,13 @@ class Pointing(object):
                 os.makedirs(run_dir, exist_ok=True)
             except OSError:
                 pass
-        self.output_prefix = get_obs_id(
-            img_type=self.pointing_type,
-            project_cycle=overall_config["project_cycle"],
-            run_counter=overall_config["run_counter"],
-            pointing_id=self.obs_id,
-            pointing_type_code=self.pointing_type_code)
+        # self.output_prefix = get_obs_id(
+        #     img_type=self.pointing_type,
+        #     project_cycle=overall_config["project_cycle"],
+        #     run_counter=overall_config["run_counter"],
+        #     pointing_id=self.obs_id,
+        #     pointing_type_code=self.pointing_type_code)
+        self.output_prefix = self.pointing_type_code+self.obs_id
         self.output_dir = os.path.join(run_dir, self.output_prefix)
         if not os.path.exists(self.output_dir):
             try:

observation_sim/config/header/ImageHeader.py

@@ -355,7 +355,7 @@ def WCS_def(xlen=9216, ylen=9232, gapy=898.0, gapx1=534, gapx2=1309, ra_ref=60,
 
 
 # TODO project_cycle is temporary， is not in header defined, delete in future
-def generatePrimaryHeader(xlen=9216, ylen=9232, pointing_id='00000001', pointing_type_code='101', ra=60, dec=-40, pixel_scale=0.074, time_pt=None, im_type='SCI', exptime=150., sat_pos=[0., 0., 0.], sat_vel=[0., 0., 0.], project_cycle=6, run_counter=0, chip_name="01"):
+def generatePrimaryHeader(xlen=9216, ylen=9232, pointing_id='00000001', pointing_type_code='101', ra=60, dec=-40, pixel_scale=0.074, time_pt=None, im_type='SCI', exptime=150., sat_pos=[0., 0., 0.], sat_vel=[0., 0., 0.], project_cycle=6, run_counter=0, chip_name="01", obstype='WIDE', dataset='csst-msc-c9-25sqdeg-v3'):
 
     # array_size1, array_size2, flux, sigma = int(argv[1]), int(argv[2]), 1000.0, 5.0
 
@@ -416,8 +416,9 @@ def generatePrimaryHeader(xlen=9216, ylen=9232, pointing_id='00000001', pointing
 
     # # OBS_id = '1'+ obs_type[im_type] + str(int(project_cycle)) + pointNum.rjust(7,'0')
     # OBS_id = '1'+ obs_type[im_type] + str(int(project_cycle)) + str(int(run_counter)).rjust(2, '0') + pointNum.rjust(5,'0')
-    OBS_id = get_obs_id(img_type=im_type, project_cycle=project_cycle, run_counter=run_counter,
-                        pointing_id=pointing_id, pointing_type_code=pointing_type_code)
+    # OBS_id = get_obs_id(img_type=im_type, project_cycle='project_cycle', run_counter=run_counter,
+    #                     pointing_id=pointing_id, pointing_type_code=pointing_type_code)
+    OBS_id = pointing_type_code+pointing_id
 
     # h_prim['OBJECT'] = str(int(project_cycle)) + pointNum.rjust(7, '0')
     h_prim['OBJECT'] = pointing_id
@@ -429,8 +430,9 @@ def generatePrimaryHeader(xlen=9216, ylen=9232, pointing_id='00000001', pointing
     # file_type = {'SCI':'SCIE', 'BIAS':'BIAS', 'DARK':'DARK', 'FLAT':'FLAT', 'CRS':'CRS', 'CRD':'CRD','CALS':'CALS','CALF':'CALF'}
     # h_prim['FILETYPE'] = file_type[im_type]
     # h_prim['FILETYPE'] = get_file_type(img_type=im_type)
-    h_prim['FILETYPE'] = im_type
-
+    # h_prim['FILETYPE'] = im_type
+    h_prim['OBSTYPE'] = obstype
+    h_prim['DATASET'] = dataset
     co = coord.SkyCoord(ra, dec, unit='deg')
 
     ra_hms = format(co.ra.hms.h, '02.0f') + format(co.ra.hms.m,
@@ -469,7 +471,7 @@ def generatePrimaryHeader(xlen=9216, ylen=9232, pointing_id='00000001', pointing
         end_time_str[11:13] + end_time_str[14:16] + end_time_str[17:19]
     # h_prim['FILENAME'] = 'CSST_MSC_MS_' + im_type + '_' + file_start_time + '_' + file_end_time + '_' + OBS_id + '_' + CCDID[
     #     k - 1].rjust(2, '0') + '_L0_V01'
-    h_prim['FILENAME'] = 'CSST_MSC_MS_' + h_prim['FILETYPE'] + '_' + \
+    h_prim['FILENAME'] = 'CSST_MSC_MS_' + im_type + '_' + \
         file_start_time + '_' + file_end_time + \
         '_' + OBS_id + '_' + chip_name + '_L0_V01'
 
@@ -531,8 +533,8 @@ def generateExtensionHeader(chip, xlen=9216, ylen=9232, ra=60, dec=-40, pa=-23.4
     # h_ext['CCDCHIP'] = CCDID[k - 1].rjust(2, '0')
     # h_ext['CCDLABEL'] = filters[k-1] + '-' + filterID[k-1]
     # h_ext['FILTER'] = filters[k-1]
-    h_ext['CHIPID'] = str(chip.chipID).rjust(2, '0')
-    h_ext['CHIPLABL'] = chip.chip_name
+    h_ext['DETECTOR'] = str(chip.chipID).rjust(2, '0')
+    h_ext['DETLABEL'] = chip.chip_name
     h_ext['FILTER'] = chip.filter_type
     h_ext['NAXIS1'] = xlen
     h_ext['NAXIS2'] = ylen

observation_sim/instruments/Filter.py

@@ -45,7 +45,6 @@ class Filter(object):
 
     def is_too_bright(self, mag, margin=-2.5):
         return mag <= self.mag_saturation + margin
-        # return mag <= 14.0
 
     def is_too_dim(self, mag, margin=1.0):
         return mag >= self.mag_limiting + margin
@@ -143,7 +142,7 @@ class Filter(object):
         del sl
         gc.collect()
 
-    def update_limit_saturation_mags(self, exptime=150., psf_fwhm=0.1969, skyFn='sky_emiss_hubble_50_50_A.dat', chip=None):
+    def update_limit_saturation_mags(self, exptime=150., psf_fwhm=0.1969, skyFn='sky_emiss_hubble_50_50_A.dat', chip=None, full_depth_exptime=None):
         if self.filter_type in _util.SPEC_FILTERS:
             return
         if chip is not None:
@@ -161,5 +160,8 @@ class Filter(object):
         self.mag_limiting, self.mag_saturation = _util.calculateLimitMag(
             psf_fwhm=psf_fwhm, pixelSize=pix_scale, throughputFn=throughput_file, readout=5.0, skyFn=skyFn, darknoise=dark_noise, exTime=exptime, fw=full_well)
 
+        if full_depth_exptime is not None:
+            self.mag_limiting, _ = _util.calculateLimitMag(psf_fwhm=psf_fwhm, pixelSize=pix_scale, throughputFn=throughput_file, readout=5.0, skyFn=skyFn, darknoise=dark_noise, exTime=full_depth_exptime, fw=full_well)
+
         print("for filter %s: mag_limiting: %.3f, mag_saturation: %.3f" %
               (self.filter_type, self.mag_limiting, self.mag_saturation))

observation_sim/instruments/FilterParam.py

@@ -25,7 +25,7 @@ class FilterParam(object):
         # 6) sky background: e/pix/s
         # 7) saturation magnitude
         # 8) dim end magnitude
-        if filter_param == None:
+        if filter_param is None:
             filtP = {
                 "NUV":   [2867.7,   705.4,  2470.0,   3270.0,  0.1404, 0.004, 15.7, 25.4],
                 "u":     [3601.1,   852.1,  3120.0,   4090.0,  0.2176, 0.021, 16.1, 25.4],
@@ -37,9 +37,9 @@ class FilterParam(object):
                 # [TODO]
                 "FGS":   [5000.0,  8000.0,  3000.0,  11000.0,  0.6500, 0.164, 0., 30.],
 
-                "GU":    [0.0,        0.0,  2550.0,   4200.0,     1.0, 0.037, 14.0, 26.0],
-                "GV":    [0.0,        0.0,  4000.0,   6500.0,     1.0, 0.037, 14.0, 26.0],
-                "GI":    [0.0,        0.0,  6200.0,  10000.0,     1.0, 0.037, 14.0, 26.0],
+                "GU":    [0.0,        0.0,  2550.0,   10000.0,     1.0, 0.037, 14.0, 26.0],
+                "GV":    [0.0,        0.0,  3500.0,   6500.0,     1.0, 0.037, 14.0, 26.0],
+                "GI":    [0.0,        0.0,  5000.0,  10000.0,     1.0, 0.037, 14.0, 26.0],
             }
         else:
             filtP = filter_param

observation_sim/instruments/FocalPlane.py

@@ -9,7 +9,7 @@ class FocalPlane(object):
         self.nchips = 42
         self.ignore_chips = []
 
-        if bad_chips == None:
+        if bad_chips is None:
             self.bad_chips = []
         else:
             self.bad_chips = bad_chips
@@ -80,7 +80,7 @@ class FocalPlane(object):
         if logger is not None:
             logger.info(
                 "    Construct the wcs of the entire image mosaic using Gnomonic/TAN projection")
-        if (xcen == None) or (ycen == None):
+        if (xcen is None) or (ycen is None):
             xcen = self.cen_pix_x
             ycen = self.cen_pix_y