Merge branch 'wcs_test' into 'develop'

starting point of the new version

See merge request csst_sim/csst-simulation!15

README.md

@@ -11,7 +11,7 @@
     * 导星探测器仿真
     * 仪器参数更新，bug修复等
 * 2022.08.23: 更新至最新的透过率曲线，在图像头文件中加入版本号。仿真软件版本号进行重新标记，当前版本为1.0.4
-* 2022.06.20: 版本v0.5上线，请于release_v0.5 branch中进行clone操作(注：目前已重新设置为v1.0，release_v1.0.0)
+* 2022.06.20: 版本v0.5上线，release_v0.5 branch(注：目前已重新设置为v1.0，release_v1.0.0)
 * 2022.04.13: 修复了天测模块中的调用错误
 * 2022.04.08: 修复了仅输出星表选项("out_cat_only")存在的问题
 

config/config_50sqdeg.yaml

+---
+###############################################
+#
+#  Configuration file for CSST simulation
+#  CSST-Sim Group, 2023/04/25
+#
+###############################################
+
+# Base diretories and naming setup
+# Can add some of the command-line arguments here as well;
+# OK to pass either way or both, as long as they are consistent
+work_dir: "/share/home/fangyuedong/csst-simulation/workplace/"
+data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
+run_name: "rotate_0"
+
+# Whether to use MPI
+run_option:
+  use_mpi: NO
+  # NOTE: "n_threads" paramters is currently not used in the backend
+  # simulation codes. It should be implemented later in the web frontend
+  # in order to config the number of threads to request from NAOC cluster
+  n_threads: 80
+
+  # Output catalog only?
+  # If yes, no imaging simulation will run
+  out_cat_only: NO
+
+###############################################
+# Catalog setting
+###############################################
+# Configure your catalog: options to be implemented 
+# in the corresponding (user defined) 'Catalog' class
+catalog_options:
+  input_path:
+    cat_dir: ""
+    # star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
+    star_cat: "starcat/"
+    galaxy_cat: "qsocat/cat2CSSTSim_bundle-50sqDeg/"
+    # AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
+
+  SED_templates_path:
+    star_SED: "Catalog_20210126/SpecLib.hdf5"
+    galaxy_SED: "Catalog_C6_20221212/sedlibs/"
+    AGN_SED: "qsocat/qsosed/"
+    # AGN_SED_WAVE: "wave_ross13.npy"
+
+  # Only simulate stars?
+  star_only: NO
+
+  # Only simulate galaxies?
+  galaxy_only: YES
+
+  # rotate galaxy ellipticity
+  rotateEll: 0. # [degree]
+
+  seed_Av: 121212    # Seed for generating random intrinsic extinction
+
+###############################################
+# Observation setting
+###############################################
+obs_setting:
+
+  # Options for survey types:
+  # "Photometric": simulate photometric chips only
+  # "Spectroscopic": simulate slitless spectroscopic chips only
+  # "FGS": simulate FGS chips only (31-42)
+  # "All": simulate full focal plane
+  survey_type: "Photometric"
+  
+  # Exposure time [seconds]
+  exp_time: 150.
+
+  # Observation starting date & time
+  date_obs: "210525" # [yymmdd]
+  time_obs: "120000" # [hhmmss]
+
+  # Default Pointing [degrees]
+  # Note: NOT valid when a pointing list file is specified
+  ra_center: 192.8595
+  dec_center:  27.1283
+  # Image rotation [degree]
+  image_rot: -113.4333
+
+  # (Optional) a file of point list 
+  # if you just want to run default pointing:
+  # - pointing_dir: null
+  # - pointing_file: null
+  pointing_dir: "/share/home/fangyuedong/50sqdeg_pointings/"
+  pointing_file: "pointing_50_combined.dat"
+
+  # Number of calibration pointings
+  np_cal: 0
+
+  # Run specific pointing(s):
+  # - give a list of indexes of pointings: [ip_1, ip_2...]
+  # - run all pointings: null
+  # Note: only valid when a pointing list is specified
+  run_pointings: [0]
+  
+  # Run specific chip(s):
+  # - give a list of indexes of chips: [ip_1, ip_2...]
+  # - run all chips: null
+  # Note: for all pointings
+  run_chips: [8]
+
+  # Whether to enable astrometric modeling
+  enable_astrometric_model: True
+
+  # Whether to enable straylight model
+  enable_straylight_model: True
+
+  # Cut by saturation magnitude in which band?
+  cut_in_band: "z"
+
+  # saturation magnitude margin
+  mag_sat_margin: -2.5
+
+  # limiting magnitude margin
+  mag_lim_margin: +1.0
+
+###############################################
+# PSF setting
+###############################################
+psf_setting:
+  
+  # Which PSF model to use:
+  # "Gauss": simple gaussian profile
+  # "Interp": Interpolated PSF from sampled ray-tracing data
+  psf_model: "Interp"
+
+  # PSF size [arcseconds]
+  # radius of 80% energy encircled
+  # NOTE: only valid for "Gauss" PSF
+  psf_rcont: 0.15
+
+  # path to PSF data
+  # NOTE: only valid for "Interp" PSF
+  psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
+
+###############################################
+# Shear setting
+###############################################
+
+shear_setting:
+  # Options to generate mock shear field:
+  # "constant": all galaxies are assigned a constant reduced shear
+  # "catalog": from catalog
+  shear_type: "catalog"
+
+  # For constant shear filed
+  reduced_g1: 0.
+  reduced_g2: 0.
+
+###############################################
+# Instrumental effects setting
+###############################################
+ins_effects:
+  # switches
+  # Note: bias_16channel, gain_16channel, and shutter_effect
+  # is currently not applicable to "FGS" observations
+  field_dist:     ON  # Whether to add field distortions
+  add_back:       ON  # Whether to add sky background
+  add_dark:       ON  # Whether to add dark noise
+  add_readout:    ON  # Whether to add read-out (Gaussian) noise
+  add_bias:       ON  # Whether to add bias-level to images
+  bias_16channel: ON  # Whether to add different biases for 16 channels
+  gain_16channel: ON  # Whether to make different gains for 16 channels
+  shutter_effect: ON  # Whether to add shutter effect
+  flat_fielding:  ON  # Whether to add flat-fielding effect
+  prnu_effect:    ON  # Whether to add PRNU effect
+  non_linear:     ON  # Whether to add non-linearity
+  cosmic_ray:     ON  # Whether to add cosmic-ray
+  cray_differ:    ON  # Whether to generate different cosmic ray maps CAL and MS output
+  cte_trail:      ON  # Whether to simulate CTE trails
+  saturbloom:     ON  # Whether to simulate Saturation & Blooming
+  add_badcolumns: ON  # Whether to add bad columns
+  add_hotpixels:  ON  # Whether to add hot pixels
+  add_deadpixels: ON  # Whether to add dead(dark) pixels
+  bright_fatter:  ON  # Whether to simulate Brighter-Fatter (also diffusion) effect
+
+  # Values: 
+  # default values have been defined individually for each chip in:
+  # ObservationSim/Instrument/data/ccd/chip_definition.json
+  # Set them here will override the default values
+  # dark_exptime:   300   # Exposure time for dark current frames [seconds]
+  # flat_exptime:   150   # Exposure time for flat-fielding frames [seconds]
+  # readout_time:   40    # The read-out time for each channel [seconds]
+  # df_strength:    2.3   # Sillicon sensor diffusion strength
+  # bias_level:     500   # bias level [e-/pixel]
+  # gain:           1.1   # Gain
+  # full_well:      90000 # Full well depth [e-]
+
+###############################################
+# Output options (for calibration pointings only)
+###############################################
+output_setting:
+  readout16:      OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
+  shutter_output: OFF # Whether to export shutter effect 16-bit image
+  bias_output:    ON  # Whether to export bias frames
+  dark_output:    ON  # Whether to export the combined dark current files
+  flat_output:    ON  # Whether to export the combined flat-fielding files
+  prnu_output:    OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
+  NBias:          1     # Number of bias frames to be exported for each exposure
+  NDark:          1     # Number of dark frames to be exported for each exposure
+  NFlat:          1     # Number of flat frames to be exported for each exposure
+
+###############################################
+# Random seeds
+###############################################
+random_seeds:
+  seed_poisson:         20210601  # Seed for Poisson noise
+  seed_CR:              20210317  # Seed for generating random cosmic ray maps
+  seed_flat:            20210101  # Seed for generating random flat fields
+  seed_prnu:            20210102  # Seed for photo-response non-uniformity
+  seed_gainNonUniform:  20210202  # Seed for gain nonuniformity
+  seed_biasNonUniform:  20210203  # Seed for bias nonuniformity
+  seed_rnNonUniform:    20210204  # Seed for readout-noise nonuniformity
+  seed_badcolumns:      20240309  # Seed for bad columns
+  seed_defective:       20210304  # Seed for defective (bad) pixels
+  seed_readout:         20210601  # Seed for read-out gaussian noise
+...
\ No newline at end of file

config/config_C6.yaml

@@ -9,13 +9,13 @@
 # Base diretories and naming setup
 # Can add some of the command-line arguments here as well;
 # OK to pass either way or both, as long as they are consistent
-work_dir: "/share/home/fangyuedong/fgs_sim/csst-simulation/workplace/"
+work_dir: "/share/home/fangyuedong/csst-simulation/workplace/"
 data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
-run_name: "C6_spectroscopic_20230225"
+run_name: "profile_C6"
 
 # Whether to use MPI
 run_option:
-  use_mpi: YES
+  use_mpi: NO
   # NOTE: "n_threads" paramters is currently not used in the backend
   # simulation codes. It should be implemented later in the web frontend
   # in order to config the number of threads to request from NAOC cluster
@@ -100,13 +100,13 @@ obs_setting:
   # - give a list of indexes of chips: [ip_1, ip_2...]
   # - run all chips: null
   # Note: for all pointings
-  run_chips: [24]
+  run_chips: [8]
 
   # Whether to enable astrometric modeling
   enable_astrometric_model: True
 
   # Whether to enable straylight model
-  enable_straylight_model: False
+  enable_straylight_model: True
 
   # Cut by saturation magnitude in which band?
   cut_in_band: "z"
@@ -144,17 +144,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "catalog"
 
   # For constant shear filed
   reduced_g1: 0.
   reduced_g2: 0.
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################
@@ -162,25 +157,25 @@ ins_effects:
   # switches
   # Note: bias_16channel, gain_16channel, and shutter_effect
   # is currently not applicable to "FGS" observations
-  field_dist:     ON  # Whether to add field distortions
-  add_back:       ON  # Whether to add sky background
-  add_dark:       ON  # Whether to add dark noise
-  add_readout:    ON  # Whether to add read-out (Gaussian) noise
-  add_bias:       ON  # Whether to add bias-level to images
-  bias_16channel: ON  # Whether to add different biases for 16 channels
-  gain_16channel: ON  # Whether to make different gains for 16 channels
-  shutter_effect: ON  # Whether to add shutter effect
-  flat_fielding:  ON  # Whether to add flat-fielding effect
-  prnu_effect:    ON  # Whether to add PRNU effect
-  non_linear:     ON  # Whether to add non-linearity
-  cosmic_ray:     ON  # Whether to add cosmic-ray
-  cray_differ:    ON  # Whether to generate different cosmic ray maps CAL and MS output
-  cte_trail:      ON  # Whether to simulate CTE trails
-  saturbloom:     ON  # Whether to simulate Saturation & Blooming
-  add_badcolumns: ON  # Whether to add bad columns
-  add_hotpixels:  ON  # Whether to add hot pixels
-  add_deadpixels: ON  # Whether to add dead(dark) pixels
-  bright_fatter:  ON  # Whether to simulate Brighter-Fatter (also diffusion) effect
+  field_dist:     YES  # Whether to add field distortions
+  add_back:       YES  # Whether to add sky background
+  add_dark:       YES  # Whether to add dark noise
+  add_readout:    YES  # Whether to add read-out (Gaussian) noise
+  add_bias:       YES  # Whether to add bias-level to images
+  bias_16channel: YES  # Whether to add different biases for 16 channels
+  gain_16channel: YES  # Whether to make different gains for 16 channels
+  shutter_effect: YES  # Whether to add shutter effect
+  flat_fielding:  YES  # Whether to add flat-fielding effect
+  prnu_effect:    YES  # Whether to add PRNU effect
+  non_linear:     YES  # Whether to add non-linearity
+  cosmic_ray:     YES  # Whether to add cosmic-ray
+  cray_differ:    YES  # Whether to generate different cosmic ray maps CAL and MS output
+  cte_trail:      YES  # Whether to simulate CTE trails
+  saturbloom:     YES  # Whether to simulate Saturation & Blooming
+  add_badcolumns: YES  # Whether to add bad columns
+  add_hotpixels:  YES  # Whether to add hot pixels
+  add_deadpixels: YES  # Whether to add dead(dark) pixels
+  bright_fatter:  YES  # Whether to simulate Brighter-Fatter (also diffusion) effect
 
   # Values: 
   # default values have been defined individually for each chip in:

config/config_C6_fits.yaml

@@ -147,17 +147,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "catalog"
 
   # For constant shear filed
   reduced_g1: 0.
   reduced_g2: 0.
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/config_C6_test_wcs.yaml

+---
+###############################################
+#
+#  Configuration file for CSST simulation
+#  CSST-Sim Group, 2023/04/25
+#
+###############################################
+
+# Base diretories and naming setup
+# Can add some of the command-line arguments here as well;
+# OK to pass either way or both, as long as they are consistent
+work_dir: "/share/home/fangyuedong/csst-simulation/workplace/"
+data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
+run_name: "fd_shear_g2m"
+
+# Whether to use MPI
+run_option:
+  use_mpi: YES
+  # NOTE: "n_threads" paramters is currently not used in the backend
+  # simulation codes. It should be implemented later in the web frontend
+  # in order to config the number of threads to request from NAOC cluster
+  n_threads: 80
+
+  # Output catalog only?
+  # If yes, no imaging simulation will run
+  out_cat_only: NO
+
+###############################################
+# Catalog setting
+###############################################
+# Configure your catalog: options to be implemented 
+# in the corresponding (user defined) 'Catalog' class
+catalog_options:
+  input_path:
+    cat_dir: "Catalog_C6_20221212"
+    star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
+    galaxy_cat: "cat2CSSTSim_bundle/"
+    AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
+
+  SED_templates_path:
+    star_SED: "Catalog_20210126/SpecLib.hdf5"
+    galaxy_SED: "Catalog_C6_20221212/sedlibs/"
+    AGN_SED: "quickspeclib_ross13.fits"
+    AGN_SED_WAVE: "wave_ross13.npy"
+
+  # Only simulate stars?
+  star_only: NO
+
+  # Only simulate galaxies?
+  galaxy_only: YES
+
+  # rotate galaxy ellipticity
+  rotateEll: 0. # [degree]
+
+  seed_Av: 121212    # Seed for generating random intrinsic extinction
+
+###############################################
+# Observation setting
+###############################################
+obs_setting:
+
+  # Options for survey types:
+  # "Photometric": simulate photometric chips only
+  # "Spectroscopic": simulate slitless spectroscopic chips only
+  # "FGS": simulate FGS chips only (31-42)
+  # "All": simulate full focal plane
+  survey_type: "All"
+  
+  # Exposure time [seconds]
+  exp_time: 150.
+
+  # Observation starting date & time
+  date_obs: "210525" # [yymmdd]
+  time_obs: "120000" # [hhmmss]
+
+  # Default Pointing [degrees]
+  # Note: NOT valid when a pointing list file is specified
+  ra_center: 192.8595
+  dec_center:  27.1283
+  # Image rotation [degree]
+  image_rot: -113.4333
+
+  # (Optional) a file of point list 
+  # if you just want to run default pointing:
+  # - pointing_dir: null
+  # - pointing_file: null
+  pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
+  pointing_file: "pointing_radec_246.5_40.dat"
+
+  # Number of calibration pointings
+  np_cal: 0
+
+  # Run specific pointing(s):
+  # - give a list of indexes of pointings: [ip_1, ip_2...]
+  # - run all pointings: null
+  # Note: only valid when a pointing list is specified
+  run_pointings: [0]
+  
+  # Run specific chip(s):
+  # - give a list of indexes of chips: [ip_1, ip_2...]
+  # - run all chips: null
+  # Note: for all pointings
+  run_chips: [9]
+
+  # Whether to enable astrometric modeling
+  enable_astrometric_model: False
+
+  # Whether to enable straylight model
+  enable_straylight_model: False
+
+  # Cut by saturation magnitude in which band?
+  cut_in_band: "z"
+
+  # saturation magnitude margin
+  mag_sat_margin: -2.5
+
+  # limiting magnitude margin
+  mag_lim_margin: +1.0
+
+###############################################
+# PSF setting
+###############################################
+psf_setting:
+  
+  # Which PSF model to use:
+  # "Gauss": simple gaussian profile
+  # "Interp": Interpolated PSF from sampled ray-tracing data
+  psf_model: "Interp"
+
+  # PSF size [arcseconds]
+  # radius of 80% energy encircled
+  # NOTE: only valid for "Gauss" PSF
+  psf_rcont: 0.15
+
+  # path to PSF data
+  # NOTE: only valid for "Interp" PSF
+  psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
+
+###############################################
+# Shear setting
+###############################################
+
+shear_setting:
+  # Options to generate mock shear field:
+  # "constant": all galaxies are assigned a constant reduced shear
+  # "catalog": from catalog
+  shear_type: "catalog"
+
+  # For constant shear filed
+  reduced_g1: 0.
+  reduced_g2: 0.
+
+###############################################
+# Instrumental effects setting
+###############################################
+ins_effects:
+  # switches
+  # Note: bias_16channel, gain_16channel, and shutter_effect
+  # is currently not applicable to "FGS" observations
+  field_dist:     YES  # Whether to add field distortions
+  add_back:       YES  # Whether to add sky background
+  add_dark:       YES  # Whether to add dark noise
+  add_readout:    YES  # Whether to add read-out (Gaussian) noise
+  add_bias:       YES  # Whether to add bias-level to images
+  bias_16channel: OFF  # Whether to add different biases for 16 channels
+  gain_16channel: OFF  # Whether to make different gains for 16 channels
+  shutter_effect: OFF  # Whether to add shutter effect
+  flat_fielding:  OFF  # Whether to add flat-fielding effect
+  prnu_effect:    OFF  # Whether to add PRNU effect
+  non_linear:     OFF  # Whether to add non-linearity
+  cosmic_ray:     OFF  # Whether to add cosmic-ray
+  cray_differ:    OFF  # Whether to generate different cosmic ray maps CAL and MS output
+  cte_trail:      OFF  # Whether to simulate CTE trails
+  saturbloom:     OFF  # Whether to simulate Saturation & Blooming
+  add_badcolumns: OFF  # Whether to add bad columns
+  add_hotpixels:  OFF  # Whether to add hot pixels
+  add_deadpixels: OFF  # Whether to add dead(dark) pixels
+  bright_fatter:  OFF  # Whether to simulate Brighter-Fatter (also diffusion) effect
+
+  # Values: 
+  # default values have been defined individually for each chip in:
+  # ObservationSim/Instrument/data/ccd/chip_definition.json
+  # Set them here will override the default values
+  # dark_exptime:   300   # Exposure time for dark current frames [seconds]
+  # flat_exptime:   150   # Exposure time for flat-fielding frames [seconds]
+  # readout_time:   40    # The read-out time for each channel [seconds]
+  # df_strength:    2.3   # Sillicon sensor diffusion strength
+  # bias_level:     500   # bias level [e-/pixel]
+  # gain:           1.1   # Gain
+  # full_well:      90000 # Full well depth [e-]
+
+###############################################
+# Output options (for calibration pointings only)
+###############################################
+output_setting:
+  readout16:      OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
+  shutter_output: OFF # Whether to export shutter effect 16-bit image
+  bias_output:    ON  # Whether to export bias frames
+  dark_output:    ON  # Whether to export the combined dark current files
+  flat_output:    ON  # Whether to export the combined flat-fielding files
+  prnu_output:    OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
+  NBias:          1     # Number of bias frames to be exported for each exposure
+  NDark:          1     # Number of dark frames to be exported for each exposure
+  NFlat:          1     # Number of flat frames to be exported for each exposure
+
+###############################################
+# Random seeds
+###############################################
+random_seeds:
+  seed_poisson:         20210601  # Seed for Poisson noise
+  seed_CR:              20210317  # Seed for generating random cosmic ray maps
+  seed_flat:            20210101  # Seed for generating random flat fields
+  seed_prnu:            20210102  # Seed for photo-response non-uniformity
+  seed_gainNonUniform:  20210202  # Seed for gain nonuniformity
+  seed_biasNonUniform:  20210203  # Seed for bias nonuniformity
+  seed_rnNonUniform:    20210204  # Seed for readout-noise nonuniformity
+  seed_badcolumns:      20240309  # Seed for bad columns
+  seed_defective:       20210304  # Seed for defective (bad) pixels
+  seed_readout:         20210601  # Seed for read-out gaussian noise
+...
\ No newline at end of file

config/config_NGP.yaml

@@ -138,17 +138,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "constant"
 
   # For constant shear filed
   reduced_g1: 0.026
   reduced_g2: 0.015
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/config_example.yaml

@@ -131,17 +131,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "constant"
 
   # For constant shear filed
   reduced_g1: 0.026
   reduced_g2: 0.015
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/config_fgs.yaml

@@ -143,17 +143,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "catalog"
 
   # For constant shear filed
   reduced_g1: 0.
   reduced_g2: 0.
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/config_testCASE.yaml

@@ -146,17 +146,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "catalog"
 
   # For constant shear filed
   reduced_g1: 0.
   reduced_g2: 0.
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/test_fd_C6.yaml

+---
+###############################################
+#
+#  Configuration file for CSST simulation
+#  CSST-Sim Group, 2023/04/25
+#
+###############################################
+
+# Base diretories and naming setup
+# Can add some of the command-line arguments here as well;
+# OK to pass either way or both, as long as they are consistent
+work_dir: "/share/home/fangyuedong/csst-simulation/workplace/"
+data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
+run_name: "fd_shear_g2m"
+
+# Whether to use MPI
+run_option:
+  use_mpi: NO
+  # NOTE: "n_threads" paramters is currently not used in the backend
+  # simulation codes. It should be implemented later in the web frontend
+  # in order to config the number of threads to request from NAOC cluster
+  n_threads: 80
+
+  # Output catalog only?
+  # If yes, no imaging simulation will run
+  out_cat_only: NO
+
+###############################################
+# Catalog setting
+###############################################
+# Configure your catalog: options to be implemented 
+# in the corresponding (user defined) 'Catalog' class
+catalog_options:
+  input_path:
+    cat_dir: "Catalog_C6_20221212"
+    star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
+    galaxy_cat: "cat2CSSTSim_bundle/"
+    AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
+
+  SED_templates_path:
+    star_SED: "Catalog_20210126/SpecLib.hdf5"
+    galaxy_SED: "Catalog_C6_20221212/sedlibs/"
+    AGN_SED: "quickspeclib_ross13.fits"
+    AGN_SED_WAVE: "wave_ross13.npy"
+
+  # Only simulate stars?
+  star_only: NO
+
+  # Only simulate galaxies?
+  galaxy_only: YES
+
+  # rotate galaxy ellipticity
+  rotateEll: 0. # [degree]
+
+  seed_Av: 121212    # Seed for generating random intrinsic extinction
+
+###############################################
+# Observation setting
+###############################################
+obs_setting:
+
+  # Options for survey types:
+  # "Photometric": simulate photometric chips only
+  # "Spectroscopic": simulate slitless spectroscopic chips only
+  # "FGS": simulate FGS chips only (31-42)
+  # "All": simulate full focal plane
+  survey_type: "Photometric"
+  
+  # Exposure time [seconds]
+  exp_time: 150.
+
+  # Observation starting date & time
+  date_obs: "210525" # [yymmdd]
+  time_obs: "120000" # [hhmmss]
+
+  # Default Pointing [degrees]
+  # Note: NOT valid when a pointing list file is specified
+  ra_center: 192.8595
+  dec_center:  27.1283
+  # Image rotation [degree]
+  image_rot: -113.4333
+
+  # (Optional) a file of point list 
+  # if you just want to run default pointing:
+  # - pointing_dir: null
+  # - pointing_file: null
+  pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
+  pointing_file: "pointing_radec_246.5_40.dat"
+
+  # Number of calibration pointings
+  np_cal: 0
+
+  # Run specific pointing(s):
+  # - give a list of indexes of pointings: [ip_1, ip_2...]
+  # - run all pointings: null
+  # Note: only valid when a pointing list is specified
+  run_pointings: [0]
+  
+  # Run specific chip(s):
+  # - give a list of indexes of chips: [ip_1, ip_2...]
+  # - run all chips: null
+  # Note: for all pointings
+  run_chips: [9]
+
+  # Whether to enable astrometric modeling
+  enable_astrometric_model: True
+
+  # Whether to enable straylight model
+  enable_straylight_model: False
+
+  # Cut by saturation magnitude in which band?
+  cut_in_band: "z"
+
+  # saturation magnitude margin
+  mag_sat_margin: -2.5
+
+  # limiting magnitude margin
+  mag_lim_margin: +1.0
+
+###############################################
+# PSF setting
+###############################################
+psf_setting:
+  
+  # Which PSF model to use:
+  # "Gauss": simple gaussian profile
+  # "Interp": Interpolated PSF from sampled ray-tracing data
+  psf_model: "Interp"
+
+  # PSF size [arcseconds]
+  # radius of 80% energy encircled
+  # NOTE: only valid for "Gauss" PSF
+  psf_rcont: 0.15
+
+  # path to PSF data
+  # NOTE: only valid for "Interp" PSF
+  psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
+
+###############################################
+# Shear setting
+###############################################
+
+shear_setting:
+  # Options to generate mock shear field:
+  # "constant": all galaxies are assigned a constant reduced shear
+  # "catalog": from catalog
+  shear_type: "catalog"
+
+  # For constant shear filed
+  reduced_g1: 0.
+  reduced_g2: 0.
+
+###############################################
+# Instrumental effects setting
+###############################################
+ins_effects:
+  # switches
+  # Note: bias_16channel, gain_16channel, and shutter_effect
+  # is currently not applicable to "FGS" observations
+  field_dist:     YES  # Whether to add field distortions
+  add_back:       YES  # Whether to add sky background
+  add_dark:       YES  # Whether to add dark noise
+  add_readout:    NO  # Whether to add read-out (Gaussian) noise
+  add_bias:       YES  # Whether to add bias-level to images
+  bias_16channel: NO  # Whether to add different biases for 16 channels
+  gain_16channel: NO  # Whether to make different gains for 16 channels
+  shutter_effect: NO  # Whether to add shutter effect
+  flat_fielding:  YES  # Whether to add flat-fielding effect
+  prnu_effect:    NO  # Whether to add PRNU effect
+  non_linear:     NO  # Whether to add non-linearity
+  cosmic_ray:     NO  # Whether to add cosmic-ray
+  cray_differ:    NO  # Whether to generate different cosmic ray maps CAL and MS output
+  cte_trail:      NO  # Whether to simulate CTE trails
+  saturbloom:     NO  # Whether to simulate Saturation & Blooming
+  add_badcolumns: NO  # Whether to add bad columns
+  add_hotpixels:  NO  # Whether to add hot pixels
+  add_deadpixels: NO  # Whether to add dead(dark) pixels
+  bright_fatter:  NO  # Whether to simulate Brighter-Fatter (also diffusion) effect
+
+  # Values: 
+  # default values have been defined individually for each chip in:
+  # ObservationSim/Instrument/data/ccd/chip_definition.json
+  # Set them here will override the default values
+  # dark_exptime:   300   # Exposure time for dark current frames [seconds]
+  # flat_exptime:   150   # Exposure time for flat-fielding frames [seconds]
+  # readout_time:   40    # The read-out time for each channel [seconds]
+  # df_strength:    2.3   # Sillicon sensor diffusion strength
+  # bias_level:     500   # bias level [e-/pixel]
+  # gain:           1.1   # Gain
+  # full_well:      90000 # Full well depth [e-]
+
+###############################################
+# Output options (for calibration pointings only)
+###############################################
+output_setting:
+  readout16:      OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
+  shutter_output: OFF # Whether to export shutter effect 16-bit image
+  bias_output:    ON  # Whether to export bias frames
+  dark_output:    ON  # Whether to export the combined dark current files
+  flat_output:    ON  # Whether to export the combined flat-fielding files
+  prnu_output:    OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
+  NBias:          1     # Number of bias frames to be exported for each exposure
+  NDark:          1     # Number of dark frames to be exported for each exposure
+  NFlat:          1     # Number of flat frames to be exported for each exposure
+
+###############################################
+# Random seeds
+###############################################
+random_seeds:
+  seed_poisson:         20210601  # Seed for Poisson noise
+  seed_CR:              20210317  # Seed for generating random cosmic ray maps
+  seed_flat:            20210101  # Seed for generating random flat fields
+  seed_prnu:            20210102  # Seed for photo-response non-uniformity
+  seed_gainNonUniform:  20210202  # Seed for gain nonuniformity
+  seed_biasNonUniform:  20210203  # Seed for bias nonuniformity
+  seed_rnNonUniform:    20210204  # Seed for readout-noise nonuniformity
+  seed_badcolumns:      20240309  # Seed for bad columns
+  seed_defective:       20210304  # Seed for defective (bad) pixels
+  seed_readout:         20210601  # Seed for read-out gaussian noise
+...
\ No newline at end of file

profile_C6.sh

+#!/bin/bash
+
+date
+
+python -m cProfile -o C6_profiler_test.pstats /share/home/fangyuedong/csst-simulation/run_sim.py \
+    --config_file config_50sqdeg.yaml \
+    --catalog C6_50sqdeg \
+    -c /share/home/fangyuedong/csst-simulation/config
+
+    # --config_file config_C6.yaml \
+    # --catalog C6_Catalog \
+    # -c /share/home/fangyuedong/csst-simulation/config
+
+    # --config_file test_fd_C6.yaml \
+    # --catalog fd_test_C6 \
+    # --config_file config_C6_test_wcs.yaml \
+    # --catalog wcs_test_C6 \

run_C6.pbs

 #!/bin/bash
-
-#PBS -N SIMS
-#PBS -l nodes=wcl-1:ppn=60
-###PBS -l nodes=wcl-1:ppn=24+wcl-2:ppn=24+wcl-3:ppn=24+wcl-4:ppn=24+wcl-5:ppn=24+wcl-6:ppn=24
+#PBS -N C6_TEST
+#PBS -l walltime=70:00:00
+#PBS -j oe
+##PBS -l nodes=wcl-1:ppn=16+wcl-2:ppn=16
+#PBS -q batch
 #PBS -u fangyuedong
-###PBS -j oe
 
 cd $PBS_O_WORKDIR
 
-NP=60
+NP=96
+hostfile=wcl-1,wcl-2
 date
 
-mpirun -np $NP python3 /share/home/fangyuedong/fgs_sim/csst-simulation/run_sim.py \
-    --config_file config_C6.yaml \
-    --catalog C6_Catalog \
-    -c /share/home/fangyuedong/fgs_sim/csst-simulation/config
+mpirun --oversubscribe -H $hostfile -np $NP python /share/home/fangyuedong/csst-simulation/run_sim.py \
+    --config_file config_50sqdeg.yaml \
+    --catalog C6_50sqdeg \
+    -c /share/home/fangyuedong/csst-simulation/config
+    # --config_file config_C6.yaml \
+    # --catalog C6_Catalog \
+    # -c /share/home/fangyuedong/csst-simulation/config
+
 

test_C6.sh

@@ -2,8 +2,8 @@
 
 date
 
-python3 /share/home/fangyuedong/fgs_sim/csst-simulation/run_sim.py \
+python3 /share/home/fangyuedong/sim_v2/csst-simulation/run_sim.py \
     --config_file config_C6.yaml \
     --catalog C6_Catalog \
-    -c /share/home/fangyuedong/fgs_sim/csst-simulation/config
+    -c /share/home/fangyuedong/sim_v2/csst-simulation/config