modified logic for defining exptime: use exptime defined in pointing as default

remove pointing_dir, data_dir in overall_config

Catalog/C6_50sqdeg.py

@@ -70,8 +70,7 @@ def get_star_cat(ra_pointing, dec_pointing):
 class Catalog(CatalogBase):
     def __init__(self, config, chip, pointing, chip_output, filt, **kwargs):
         super().__init__()
-        self.cat_dir = os.path.join(config["data_dir"], config["catalog_options"]["input_path"]["cat_dir"])
-        # self.seed_Av = config["catalog_options"]["seed_Av"]
+        self.cat_dir = config["catalog_options"]["input_path"]["cat_dir"]
 
         self.cosmo = FlatLambdaCDM(H0=67.66, Om0=0.3111)
 
@@ -92,20 +91,19 @@ class Catalog(CatalogBase):
             # Get the cloest star catalog file
             star_file_name = get_star_cat(ra_pointing=self.pointing.ra, dec_pointing=self.pointing.dec)
             star_path = os.path.join(config["catalog_options"]["input_path"]["star_cat"], star_file_name)
-            star_SED_file = config["catalog_options"]["SED_templates_path"]["star_SED"]
             self.star_path = os.path.join(self.cat_dir, star_path)
-            self.star_SED_path = os.path.join(config["data_dir"], star_SED_file)
+            self.star_SED_path = config["catalog_options"]["SED_templates_path"]["star_SED"]
             self._load_SED_lib_star()
         
         if "galaxy_cat" in config["catalog_options"]["input_path"] and config["catalog_options"]["input_path"]["galaxy_cat"] and not config["catalog_options"]["star_only"]:
             galaxy_dir = config["catalog_options"]["input_path"]["galaxy_cat"]
             self.galaxy_path = os.path.join(self.cat_dir, galaxy_dir)
-            self.galaxy_SED_path = os.path.join(config["data_dir"], config["catalog_options"]["SED_templates_path"]["galaxy_SED"])
+            self.galaxy_SED_path = config["catalog_options"]["SED_templates_path"]["galaxy_SED"]
             self._load_SED_lib_gals()
             self.agn_seds = {}
 
         if "AGN_SED" in config["catalog_options"]["SED_templates_path"] and not config["catalog_options"]["star_only"]:
-            self.AGN_SED_path = os.path.join(config["data_dir"], config["catalog_options"]["SED_templates_path"]["AGN_SED"])
+            self.AGN_SED_path = config["catalog_options"]["SED_templates_path"]["AGN_SED"]
 
         if "rotateEll" in config["catalog_options"]:
             self.rotation = np.radians(float(config["catalog_options"]["rotateEll"]))

ObservationSim/Config/Config.py

@@ -12,15 +12,19 @@ def config_dir(config, work_dir=None, data_dir=None):
         path_dict["work_dir"] = work_dir
     
     # Data directory
-    if data_dir == None:
-        # Assume all input datasets are in the work directory
-        path_dict["data_dir"] =os.path.join(path_dict["work_dir"], "data/")
-    else:
-        path_dict["data_dir"] = data_dir
+    # if data_dir == None:
+    #     # Assume all input datasets are in the work directory
+    #     path_dict["data_dir"] =os.path.join(path_dict["work_dir"], "data/")
+    # else:
+    #     path_dict["data_dir"] = data_dir
+    
     # PSF data directory
+    # if config["psf_setting"]["psf_model"] == "Interp":
+    #     path_dict["psf_pho_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_pho_dir"])
+    #     path_dict["psf_sls_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_sls_dir"])
     if config["psf_setting"]["psf_model"] == "Interp":
-        path_dict["psf_pho_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_pho_dir"])
-        path_dict["psf_sls_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_sls_dir"])
+        path_dict["psf_pho_dir"] = config["psf_setting"]["psf_pho_dir"]
+        path_dict["psf_sls_dir"] = config["psf_setting"]["psf_sls_dir"]
 
     return path_dict
 

ObservationSim/Config/Pointing.py

-import numpy as np
-import galsim
+import os
+import shutil
 import yaml
+import galsim
+import numpy as np
 from astropy.time import Time
 
+from ObservationSim.Config._util import get_obs_id
 import ObservationSim.Instrument._util as _util
 
 class Pointing(object):
@@ -103,3 +106,32 @@ class Pointing(object):
             self.pointing_type = self.obs_param["obs_type"]
         else:
             self.timestamp = t
+
+    def make_output_pointing_dir(self, overall_config, pointing_ID=0, copy_obs_config=False):
+        run_dir = os.path.join(overall_config["work_dir"], overall_config["run_name"])
+        if not os.path.exists(run_dir):
+            try:
+                os.makedirs(run_dir, exist_ok=True)
+            except OSError:
+                pass
+        # self.prefix = "MSC_" + str(pointing_ID).rjust(8, '0')
+        self.output_prefix = get_obs_id(
+            img_type=self.pointing_type, 
+            project_cycle=overall_config["project_cycle"], 
+            run_counter=overall_config["run_counter"], 
+            pointing_id=self.obs_id, 
+            pointing_type_code = self.pointing_type_code)
+        self.output_dir = os.path.join(run_dir, self.output_prefix)
+        if not os.path.exists(self.output_dir):
+            try:
+                os.makedirs(self.output_dir, exist_ok=True)
+            except OSError:
+                pass
+        if copy_obs_config and self.obs_config_file:
+            obs_config_output_path = os.path.join(self.output_dir, os.path.basename(self.obs_config_file))
+            if not os.path.exists(obs_config_output_path):
+                try:
+                    shutil.copy(args.obs_config_file, self.output_dir)
+                except OSError:
+                    pass
+

ObservationSim/ObservationSim.py

@@ -12,7 +12,7 @@ from ObservationSim.Config import config_dir, ChipOutput
 from ObservationSim.Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip
 from ObservationSim.Instrument.Chip import Effects
 from ObservationSim.Instrument.Chip import ChipUtils as chip_utils
-from ObservationSim._util import makeSubDir_PointingList
+from ObservationSim._util import make_output_pointing_dir
 from ObservationSim.Astrometry.Astrometry_util import on_orbit_obs_position
 from ObservationSim.sim_steps import SimSteps, SIM_STEP_TYPES
 
@@ -99,7 +99,6 @@ class Observation(object):
         sim_steps = SimSteps(overall_config=self.config, chip_output=chip_output, all_filters=self.all_filters)
 
         for step in pointing.obs_param["call_sequence"]:
-            print(step)
             if self.config["run_option"]["out_cat_only"]:
                 if step != "scie_obs":
                     continue
@@ -268,6 +267,9 @@ class Observation(object):
             # Construct chips & filters:
             pointing = pointing_list[ipoint]
             pointing_ID = pointing.id
+
+            sub_img_dir, prefix = make_output_pointing_dir(path_dict=self.path_dict, config=self.config, pointing_ID=pointing_ID)
+
             self.focal_plane = FocalPlane(chip_list=pointing.obs_param["run_chips"])
             # Make Chip & Filter lists
             self.chip_list = []
@@ -310,8 +312,6 @@ class Observation(object):
                         continue
                 pid = os.getpid()
 
-                sub_img_dir, prefix = makeSubDir_PointingList(path_dict=self.path_dict, config=self.config, pointing_ID=pointing_ID)
-
                 chip = run_chips[ichip]
                 filt = run_filts[ichip]
                 # chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid))

ObservationSim/_util.py

@@ -37,8 +37,12 @@ def generate_pointing_list(config, pointing_filename=None, data_dir=None):
     else:
         obs_config_file = None
     
-    if pointing_filename and data_dir:
-        pointing_file = os.path.join(data_dir, pointing_filename)
+    # if pointing_filename and data_dir:
+    if pointing_filename:
+        if data_dir:
+            pointing_file = os.path.join(data_dir, pointing_filename)
+        else:
+            pointing_file = pointing_filename
         f = open(pointing_file, 'r')
         # for _ in range(1):
         #     header = f.readline()
@@ -93,7 +97,7 @@ def make_run_dirs(work_dir, run_name, pointing_list):
             pass
     return imgDir
 
-def makeSubDir_PointingList(path_dict, config, pointing_ID=0):
+def make_output_pointing_dir(path_dict, config, pointing_ID=0):
     imgDir = os.path.join(path_dict["work_dir"], config["run_name"])
     if not os.path.exists(imgDir):
         try:

ObservationSim/sim_steps/add_cosmic_rays.py

@@ -2,10 +2,17 @@ from ObservationSim.Instrument.Chip import ChipUtils as chip_utils
 
 def add_cosmic_rays(self, chip, filt, tel, pointing, catalog, obs_param):
     self.chip_output.Log_info("  Adding Cosmic-Ray")
+    
+    # Get exposure time
+    if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None):
+        exptime = obs_param["exptime"]
+    else:
+        exptime = pointing.exp_time
+
     chip.img, crmap_gsimg, cr_event_num = chip_utils.add_cosmic_rays(
         img=chip.img, 
         chip=chip, 
-        exptime=pointing.exp_time, 
+        exptime=exptime, 
         seed=self.overall_config["random_seeds"]["seed_CR"]+pointing.id*30+chip.chipID)
     # [TODO] output cosmic ray image
     return chip, filt, tel, pointing
\ No newline at end of file

ObservationSim/sim_steps/add_objects.py

@@ -12,6 +12,12 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
     # Prepare output file(s) for this chip
     self.chip_output.create_output_file()
 
+    # Get exposure time
+    if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None):
+        exptime = obs_param["exptime"]
+    else:
+        exptime = pointing.exp_time
+
     # Load catalogues
     if catalog is None:
         self.chip_output.Log_error("Catalog interface class must be specified for SCIE-OBS")
@@ -146,7 +152,7 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
                     chip=chip, 
                     g1=obj.g1, 
                     g2=obj.g2, 
-                    exptime=pointing.exp_time,
+                    exptime=exptime,
                     fd_shear=fd_shear)
 
             elif chip.survey_type == "spectroscopic" and not self.overall_config["run_option"]["out_cat_only"]:
@@ -159,7 +165,7 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
                     chip=chip, 
                     g1=obj.g1, 
                     g2=obj.g2, 
-                    exptime=pointing.exp_time,
+                    exptime=exptime,
                     normFilter=norm_filt,
                     fd_shear=fd_shear)
 

ObservationSim/sim_steps/add_pattern_noise.py

@@ -11,20 +11,27 @@ def apply_PRNU(self, chip, filt, tel, pointing, catalog, obs_param):
 def add_poisson_and_dark(self, chip, filt, tel, pointing, catalog, obs_param):
     # Add dark current & Poisson noise
     InputDark = False
+    
+    # Get exposure time
+    if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None):
+        exptime = obs_param["exptime"]
+    else:
+        exptime = pointing.exp_time
+    
     if obs_param["add_dark"] == True:
         if InputDark:
             chip.img = chip_utils.add_inputdark(img=chip.img, 
                                                chip=chip, 
-                                               exptime=pointing.exp_time)
+                                               exptime=exptime)
         else:
             chip.img, _ = chip_utils.add_poisson(img=chip.img, 
                                                 chip=chip, 
-                                                exptime=pointing.exp_time, 
+                                                exptime=exptime, 
                                                 poisson_noise=chip.poisson_noise)
     else:
         chip.img, _ = chip_utils.add_poisson(img=chip.img, 
                                             chip=self, 
-                                            exptime=pointing.exp_time, 
+                                            exptime=exptime, 
                                             poisson_noise=chip.poisson_noise, 
                                             dark_noise=0.)
     return chip, filt, tel, pointing

ObservationSim/sim_steps/add_sky_background.py

@@ -4,6 +4,13 @@ from ObservationSim.Straylight import calculateSkyMap_split_g
 from ObservationSim.Instrument import FilterParam
 
 def add_sky_background_sci(self, chip, filt, tel, pointing, catalog, obs_param):
+    
+    # Get exposure time
+    if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None):
+        exptime = obs_param["exptime"]
+    else:
+        exptime = pointing.exp_time
+    
     flat_normal = np.ones_like(chip.img.array)
     if obs_param["flat_fielding"] == True:
         flat_normal = flat_normal * chip.flat_img.array / np.mean(chip.flat_img.array)
@@ -25,7 +32,7 @@ def add_sky_background_sci(self, chip, filt, tel, pointing, catalog, obs_param):
         self.chip_output.Log_info("sky background + stray light pixel flux value: %.5f"%(filt.sky_background))
     
     if chip.survey_type == "photometric":
-        sky_map = filt.getSkyNoise(exptime = obs_param["exptime"])
+        sky_map = filt.getSkyNoise(exptime = exptime)
         sky_map = sky_map * np.ones_like(chip.img.array) * flat_normal
         sky_map = galsim.Image(array=sky_map)
     else:
@@ -39,7 +46,7 @@ def add_sky_background_sci(self, chip, filt, tel, pointing, catalog, obs_param):
                 isAlongY=0,
                 flat_cube=chip.flat_cube, 
                 zoldial_spec = filt.zodical_spec)
-        sky_map = (sky_map + filt.sky_background)*obs_param["exptime"]
+        sky_map = (sky_map + filt.sky_background)*exptime
     
     # sky_map = sky_map * tel.pupil_area * obs_param["exptime"]
     chip.img += sky_map
@@ -51,12 +58,17 @@ def add_sky_flat_calibration(self, chip, filt, tel, pointing, catalog, obs_param
         _, _ = self.prepare_headers(chip=chip, pointing=pointing)
     chip_wcs = galsim.FitsWCS(header = self.h_ext)
 
-    expTime = obs_param["exptime"]
+    # Get exposure time
+    if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None):
+        exptime = obs_param["exptime"]
+    else:
+        exptime = pointing.exp_time
+    
     skyback_level = obs_param["flat_level"]
 
     filter_param = FilterParam()
     sky_level_filt = obs_param["flat_level_filt"]
-    norm_scaler = skyback_level/expTime / filter_param.param[sky_level_filt][5]
+    norm_scaler = skyback_level/exptime / filter_param.param[sky_level_filt][5]
 
     flat_normal = np.ones_like(chip.img.array)
     if obs_param["flat_fielding"] == True:
@@ -73,7 +85,7 @@ def add_sky_flat_calibration(self, chip, filt, tel, pointing, catalog, obs_param
     
 
     if chip.survey_type == "photometric":
-        sky_map = flat_normal * np.ones_like(chip.img.array) * norm_scaler * filter_param.param[chip.filter_type][5] / tel.pupil_area * expTime
+        sky_map = flat_normal * np.ones_like(chip.img.array) * norm_scaler * filter_param.param[chip.filter_type][5] / tel.pupil_area * exptime
     elif chip.survey_type == "spectroscopic":
         # flat_normal = np.ones_like(chip.img.array)
         if obs_param["flat_fielding"] == True:
@@ -91,7 +103,7 @@ def add_sky_flat_calibration(self, chip, filt, tel, pointing, catalog, obs_param
             pixelSize=chip.pix_scale,
             isAlongY=0,
             flat_cube=chip.flat_cube)
-        sky_map = sky_map * norm_scaler * expTime
+        sky_map = sky_map * norm_scaler * exptime
     
     chip.img += sky_map
     return chip, filt, tel, pointing

config/config_overall.yaml

@@ -11,7 +11,7 @@
 # can add some of the command-line arguments here as well;
 # ok to pass either way or both, as long as they are consistent
 work_dir: "/public/home/fangyuedong/project/workplace/"
-data_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/"
+# data_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/"
 run_name: "QSO_50sqdeg_test"
 
 # Project cycle and run counter are used to name the outputs
@@ -35,15 +35,15 @@ run_option:
 catalog_options:
   input_path:
     # cat_dir: "Catalog_C6_20221212"
-    cat_dir: ""
+    cat_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/"
     star_cat: "starcat/"
     galaxy_cat: "qsocat/cat2CSSTSim_bundle-50sqDeg/"
     # AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
 
   SED_templates_path:
-    star_SED: "SpecLib.hdf5"
-    galaxy_SED: "sedlibs/"
-    AGN_SED: "qsocat/qsosed/"
+    star_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/SpecLib.hdf5"
+    galaxy_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/sedlibs/"
+    AGN_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/qsocat/qsosed/"
     # AGN_SED_WAVE: "wave_ross13.npy"
 
   # Only simulate stars?
@@ -52,11 +52,6 @@ catalog_options:
   # Only simulate galaxies?
   galaxy_only: YES
 
-  # # rotate galaxy ellipticity
-  # rotateEll: 0. # [degree]
-
-  # seed_Av: 121212    # Seed for generating random intrinsic extinction
-
 ###############################################
 # Observation setting
 ###############################################
@@ -65,8 +60,8 @@ obs_setting:
   # if you just want to run default pointing:
   # - pointing_dir: null
   # - pointing_file: null
-  pointing_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg"
-  pointing_file: "pointing_50_combined.dat"
+  # pointing_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/pointing50_C9/"
+  pointing_file: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/pointing50_C9/pointing_50_1_n.dat"
 
   obs_config_file: "/public/home/fangyuedong/project/csst-simulation/config/obs_config_SCI_WIDE_phot.yaml"
 

config/obs_config_SCI_WIDE_phot.yaml

@@ -11,7 +11,7 @@
 # Observation type
 obs_type: "SCI"
 obs_type_code: "101"
-obs_id: "00000001"
+obs_id: "00000001" # this setting will only be used if pointing list file is not given
 
 # Define list of chips
 run_chips: [8]
@@ -20,27 +20,37 @@ run_chips: [8]
 call_sequence:
   # Accumulate fluxes from objects
   scie_obs:
-    exptime: 150. # [s]
+    # [Optional]: exposure time of the pointing will be used as default.
+    # Set it here is you want to override the default
+    # exptime: 150. # [s] 
     shutter_effect: YES
     flat_fielding: YES
     field_dist: YES
   # Accumulate fluxes from sky background
-  # flat_level: set the total skybackground value (e-) in the exptime,if none,set null, or delete the key
-  # flat_level_filt: the vale of "flat_level" is in the filter "flat_level_filt", can set NUV, u, g, r, i, z, y, if 
-  # none,set null,or delete the key
   sky_background:
-    exptime: 150. # [s]
+    # [Optional]: exposure time of the pointing will be used as default.
+    # Set it here is you want to override the default
+    # exptime: 150. # [s]
     shutter_effect: YES
     flat_fielding: YES
     enable_straylight_model: YES
+    # flat_level: set the total skybackground value (e-) in the exptime,if none,set null, or delete the key
+    # flat_level_filt: the vale of "flat_level" is in the filter "flat_level_filt", can set NUV, u, g, r, i, z, y, if 
+    # none,set null,or delete the key
     flat_level: null
     flat_level_filt: null
   # Apply PRNU to accumulated photons
   PRNU_effect: {}
   # Accumulate photons caused by cosmic rays
-  cosmic_rays: {}
+  cosmic_rays: 
+    # [Optional]: exposure time of the pointing will be used as default.
+    # Set it here is you want to override the default
+    # exptime: 150. # [s]
   # Add Poission noise and dark current
   poisson_and_dark:
+    # [Optional]: exposure time of the pointing will be used as default.
+    # Set it here is you want to override the default
+    # exptime: 150. # [s]
     add_dark: YES
   # Simulate brighter fatter effects
   bright_fatter: {}

run_sim.py

@@ -49,6 +49,9 @@ def run_sim():
     if args.work_dir is not None:
         config['work_dir'] = args.work_dir
 
+    if not ("data_dir" in config):
+        config["data_dir"] = None
+
     # Some default values
     if "mag_sat_margin" not in config["obs_setting"]:
         config["obs_setting"]["mag_sat_margin"] = -2.5
@@ -64,7 +67,10 @@ def run_sim():
     # "config['obs_setting']['np_cal']"" is the number of CAL pointings which will be 
     # appended to the front.
     # NOTE: the implementation of gerenating time_stamps is temporary.
-    pointing_list = generate_pointing_list(config=config, pointing_filename=config['obs_setting']['pointing_file'], data_dir=config['obs_setting']['pointing_dir'])
+    pointing_dir = None
+    if "pointing_dir" in config['obs_setting']:
+        pointing_dir = config['obs_setting']["pointing_dir"]
+    pointing_list = generate_pointing_list(config=config, pointing_filename=config['obs_setting']['pointing_file'], data_dir=pointing_dir)
 
     # Make the main output directories
     run_dir = make_run_dirs(work_dir=config['work_dir'], run_name=config['run_name'], pointing_list=pointing_list)

tests/PSFInterpTest/test_PSFInterpModule_coverage.py

@@ -24,7 +24,7 @@ def defineCCD(iccd, config_file):
             #    print (key + " : " + str(value))
         except yaml.YAMLError as exc:
             print(exc)
-    path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir'])
+    # path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir'])
     chip = Chip(chipID=iccd, config=config)
     #chip = Chip(chipID=iccd, ccdEffCurve_dir=path_dict["ccd_dir"], CRdata_dir=path_dict["CRdata_dir"], normalize_dir=path_dict["normalize_dir"], sls_dir=path_dict['sls_dir'], config=config)
     return chip

tests/PSFInterpTest/test_loadPSFSet.py

@@ -21,7 +21,7 @@ def defineCCD(iccd, config_file):
             #    print (key + " : " + str(value))
         except yaml.YAMLError as exc:
             print(exc)
-    path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir'])
+    # path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir'])
     chip = Chip(chipID=iccd, config=config)
     #chip = Chip(chipID=iccd, ccdEffCurve_dir=path_dict["ccd_dir"], CRdata_dir=path_dict["CRdata_dir"], normalize_dir=path_dict["normalize_dir"], sls_dir=path_dict['sls_dir'], config=config)
     return chip