first commit of user-customizable catalog version

ObservationSim/_util.py

@@ -15,6 +15,83 @@ def parse_args():
     parser.add_argument('-w', '--work_dir', help='The path for output.')
     return parser.parse_args()
 
+def generate_pointings(config, pointing_filename=None, data_dir=None):
+    pRA = []
+    pDEC = []
+    if pointing_filename and data_dir:
+        pointing_file = os.path.join(data_dir, pointing_filename)
+        f = open(pointing_file, 'r')
+        for _ in range(1):
+            header = f.readline()
+        iline = 0
+        for line in f:
+            line = line.strip()
+            columns = line.split()
+            pRA.append(float(columns[0]))
+            pDEC.append(float(columns[1]))
+        f.close()
+    else:
+        pRA.append(config["obs_setting"]["ra_center"])
+        pDEC.append(config["obs_setting"]["dec_center"])
+    pRA = np.array(pRA)
+    pDEC = np.array(pDEC)
+
+    # Create calibration pointings
+    # NOTE: temporary implementation
+    ncal = config['obs_setting']['np_cal']
+    pointing_type = ['MS']*len(pRA)
+    pRA = np.append([pRA[0]]*ncal, pRA)
+    pDEC = np.append([pDEC[0]]*ncal, pDEC)
+    pointing_type = ['CAL']*ncal + pointing_type
+
+    # Calculate starting time(s)
+    # NOTE: temporary implementation
+    t0 = datetime(2021, 5, 25, 12, 0, 0)
+    t = datetime.timestamp(t0)
+    timestamp_obs = []
+    delta_t = 10 # Time elapsed between exposures (minutes)
+    for i in range(len(pointing_type)):
+        timestamp_obs.append(t)
+        if pointing_type[i] == 'CAL':
+            t += 3 * delta_t * 60 # 3 calibration exposures for each pointing
+        elif pointing_type[i] == 'MS':
+            t += delta_t * 60
+    timestamp_obs = np.array(timestamp_obs)
+    pointing_type = np.array(pointing_type)
+
+    if config['obs_setting']['run_pointings'] is None:
+        pRange = list(range(len(pRA)))
+    else:
+        ncal = config['obs_setting']['np_cal']
+        plist = config['obs_setting']['run_pointings']
+        pRange = list(range(ncal)) + [x + ncal for x in plist]
+    return pRA, pDEC, timestamp_obs, pointing_type, pRange
+
+def make_run_dirs(work_dir, run_name, nPointings, pRange=None):
+    if not os.path.exists(work_dir):
+        try:
+            os.makedirs(work_dir, exist_ok=True)
+        except OSError:
+            pass
+    imgDir = os.path.join(work_dir, run_name)
+    if not os.path.exists(imgDir):
+        try:
+            os.makedirs(imgDir, exist_ok=True)
+        except OSError:
+            pass
+    prefix = "MSC_"
+    for pointing_ID in range(nPointings):
+        if pRange is not None:
+            if pointing_ID not in pRange:
+                continue
+        fname=prefix + str(pointing_ID).rjust(7, '0')
+        subImgDir = os.path.join(imgDir, fname)
+        if not os.path.exists(subImgDir):
+            try:
+                os.makedirs(subImgDir, exist_ok=True)
+            except OSError:
+                pass
+
 def imgName(tt=0):
     ut = datetime.utcnow()
     eye, emo, eda, eho, emi, ese = str(ut.year), str(ut.month), str(ut.day), str(ut.hour), str(ut.minute), str(ut.second)

config/config_sim.yaml → config/config_C3.yaml

@@ -12,7 +12,7 @@
 # work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/"
 work_dir: "/public/home/fangyuedong/20211203/CSST/workplace/"
 data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
-run_name: "TEST"
+run_name: "C3_20211216"
 
 # (Optional) a file of point list 
 # if you just want to run default pointing:
@@ -24,7 +24,10 @@ pointing_file: "pointing10_20210202.dat"
 # Whether to use MPI
 run_option:
   use_mpi: YES
-  n_cores: 40
+  # NOTE: "n_threads" paramters is currently not used in the backend
+  # simulation codes. It should be implemented later in the web frontend
+  # in order to config the number of threads to request from NAOC cluster
+  n_threads: 40
 
 # Output catalog only?
 # If yes, no imaging simulation will run
@@ -65,19 +68,19 @@ obs_setting:
 
   # Number of calibration pointings
   # Note: only valid when a pointing list is specified
-  np_cal: 0
+  np_cal: 3
 
   # Run specific pointing(s):
   # - give a list of indexes of pointings: [ip_1, ip_2...]
   # - run all pointings: null
   # Note: only valid when a pointing list is specified
-  run_pointings: [5, 6]
+  run_pointings: [1, 2, 3, 4, 5]
   
   # Run specific chip(s):
   # - give a list of indexes of chips: [ip_1, ip_2...]
   # - run all chips: null
   # Note: for all pointings
-  run_chips: [1, 25]
+  run_chips: [1, 26, 29, 6, 7, 22, 23, 19, 20]
 
 ###############################################
 # Input path setting

config/config_example.yaml

+---
+###############################################
+#
+#  Configuration file for CSST simulation
+#  CSST-Sim Group, 2021/10/07, version 0.3
+#
+###############################################
+
+# Base diretories and naming setup
+# Can add some of the command-line arguments here as well;
+# OK to pass either way or both, as long as they are consistent
+# work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/"
+work_dir: "/public/home/fangyuedong/20211203/CSST/workplace/"
+data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
+run_name: "example_20211217"
+
+# (Optional) a file of point list 
+# if you just want to run default pointing:
+# - pointing_dir: null
+# - pointing_file: null
+pointing_dir: null
+pointing_file: null
+
+# Whether to use MPI
+run_option:
+  use_mpi: YES
+  # NOTE: "n_threads" paramters is currently not used in the backend
+  # simulation codes. It should be implemented later in the web frontend
+  # in order to config the number of threads to request from NAOC cluster
+  n_threads: 40
+
+# Output catalog only?
+# If yes, no imaging simulation will run
+out_cat_only: NO
+
+# Only simulate stars?
+star_only: NO
+
+# Only simulate galaxies?
+galaxy_only: NO
+
+###############################################
+# Observation setting
+###############################################
+obs_setting:
+
+  # Options for survey types:
+  # "Photometric": simulate photometric chips only
+  # "Spectroscopic": simulate slitless spectroscopic chips only
+  # "All": simulate full focal plane
+  survey_type: "All"
+  
+  # Exposure time [seconds]
+  exp_time: 150.
+
+  # Observation starting date & time
+  # (Subject to change)
+  date_obs: "210525" # [yymmdd]
+  time_obs: "120000" # [hhmmss]
+
+  # Default Pointing [degrees]
+  # Note: NOT valid when a pointing list file is specified
+  ra_center: 60.9624
+  dec_center:  -41.5032
+
+  # Image rotation [degree]
+  image_rot: -113.4333
+
+  # Number of calibration pointings
+  np_cal: 2
+
+  # Run specific pointing(s):
+  # - give a list of indexes of pointings: [ip_1, ip_2...]
+  # - run all pointings: null
+  # Note: only valid when a pointing list is specified
+  run_pointings: null
+  
+  # Run specific chip(s):
+  # - give a list of indexes of chips: [ip_1, ip_2...]
+  # - run all chips: null
+  # Note: for all pointings
+  run_chips: [18]
+
+###############################################
+# Input path setting
+###############################################
+
+# Default path settings for WIDE survey simulation
+input_path:
+  cat_dir: "catalog_points_7degree2/point_RA60.9624_DE-41.5032/"
+  star_cat: "stars_ccd13_p_RA60.9624_DE-41.5032.hdf5"
+  galaxy_cat: null
+
+SED_templates_path:
+  star_SED: "SED_MMW_Gaia_Cluster_D20_SS.hdf5"
+  galaxy_SED: null
+
+# TODO: should the following path settings be made hidden from user?
+Efficiency_curve_path:
+  filter_eff: "Filters/"
+  ccd_eff: "Filter_CCD_Mirror/ccd/"
+  mirror_eff: "Filter_CCD_Mirror/mirror_ccdnote.txt"
+
+CR_data_path: "CRdata/"
+sky_data_path: "skybackground/sky_emiss_hubble_50_50_A.dat"
+
+SLS_path:
+  SLS_conf: "CONF"
+  SLS_norm: "normalize_filter"
+
+
+###############################################
+# PSF setting
+###############################################
+psf_setting:
+  
+  # Which PSF model to use:
+  # "Gauss": simple gaussian profile
+  # "Interp": Interpolated PSF from sampled ray-tracing data
+  psf_model: "Interp"
+
+  # PSF size [arcseconds]
+  # radius of 80% energy encircled
+  # NOTE: only valid for "Gauss" PSF
+  psf_rcont: 0.15
+
+  # path to PSF data
+  # NOTE: only valid for "Interp" PSF
+  psf_dir: "csstPSFdata/CSSOS_psf_20210108/CSST_psf_ciomp_2p5um_cycle3_ccr90"
+
+  # path to field-distortion model
+  # Note: only valid when ins_effects: field_dist is "ON"
+  fd_path: "FieldDistModelv2.0.pickle"
+  
+  # sigma_spin: 0.0 # psf spin?
+
+###############################################
+# Shear setting
+###############################################
+
+shear_setting:
+  # Options to generate mock shear field:
+  # "constant": all galaxies are assigned a constant reduced shear
+  # "catalog": from catalog (not available yet)
+  # "extra": from seprate file
+  shear_type: "constant"
+
+  # For constant shear filed
+  reduced_g1: 0.026
+  reduced_g2: 0.015
+
+  # Representation of the shear vector?
+  reShear: "E"
+
+  # rotate galaxy ellipticity
+  rotateEll: 0. # [degree]
+
+  # Extra shear catalog
+  # (currently not used)
+  # shear_cat: "mockShear.cat"
+
+###############################################
+# Instrumental effects setting
+###############################################
+ins_effects:
+  # switches
+  field_dist:     ON  # Whether to add field distortions
+  add_back:       ON  # Whether to add sky background
+  add_dark:       ON  # Whether to add dark noise
+  add_readout:    ON  # Whether to add read-out (Gaussian) noise
+  add_bias:       ON  # Whether to add bias-level to images
+  shutter_effect: ON  # Whether to add shutter effect
+  flat_fielding:  ON  # Whether to add flat-fielding effect
+  prnu_effect:    ON  # Whether to add PRNU effect
+  non_linear:     OFF # Whether to add non-linearity
+  cosmic_ray:     ON  # Whether to add cosmic-ray
+  cray_differ:    ON  # Whether to generate different cosmic ray maps CAL and MS output
+  cte_trail:      ON  # Whether to simulate CTE trails
+  saturbloom:     ON  # Whether to simulate Saturation & Blooming
+  add_badcolumns: ON  # Whether to add bad columns
+  add_hotpixels:  ON  # Whether to add hot pixels
+  add_deadpixels: ON  # Whether to add dead(dark) pixels
+  bright_fatter:  ON  # Whether to simulate Brighter-Fatter (also diffusion) effect
+
+  # values
+  dark_exptime:   300   # Exposure time for dark current frames [seconds]
+  flat_exptime:   150   # Exposure time for flat-fielding frames [seconds]
+  readout_time:   40    # The read-out time for each channel [seconds]
+  df_strength:    2.3   # Sillicon sensor diffusion strength
+  bias_level:     500   # bias level [e-/pixel]
+  gain:           1.1   # Gain
+  full_well:      90000 # Full well depth [e-]
+  NBias:          1     # Number of bias frames to be exported for each exposure
+  NDark:          1     # Number of dark frames to be exported for each exposure
+  NFlat:          1     # Number of flat frames to be exported for each exposure
+
+###############################################
+# Output options
+###############################################
+output_setting:
+  readout16:      OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
+  shutter_output: OFF # Whether to export shutter effect 16-bit image
+  bias_output:    ON  # Whether to export bias frames
+  dark_output:    ON  # Whether to export the combined dark current files
+  flat_output:    ON  # Whether to export the combined flat-fielding files
+  prnu_output:    OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
+
+###############################################
+# Random seeds
+###############################################
+random_seeds:
+  seed_Av:              121212    # Seed for generating random intrinsic extinction
+  seed_poisson:         20210601  # Seed for Poisson noise
+  seed_CR:              20210317  # Seed for generating random cosmic ray maps
+  seed_flat:            20210101  # Seed for generating random flat fields
+  seed_prnu:            20210102  # Seed for photo-response non-uniformity
+  seed_gainNonUniform:  20210202  # Seed for gain nonuniformity
+  seed_biasNonUniform:  20210203  # Seed for bias nonuniformity
+  seed_rnNonUniform:    20210204  # Seed for readout-noise nonuniformity
+  seed_badcolumns:      20240309  # Seed for bad columns
+  seed_defective:       20210304  # Seed for defective (bad) pixels
+  seed_readout:         20210601  # Seed for read-out gaussian noise
+...
\ No newline at end of file