modify codes to test straylight model crashing issue

ObservationSim/ObservationSim.py

@@ -48,7 +48,7 @@ class Observation(object):
                 self.filter_list.append(filt)
             self.all_filter.append(filt)
 
-    def run_one_chip(self, chip, filt, pointing, chip_output, wcs_fp=None, psf_model=None, shear_cat_file=None, cat_dir=None, sed_dir=None):
+    def run_one_chip(self, chip, filt, pointing, chip_output, wcs_fp=None, psf_model=None, cat_dir=None, sed_dir=None):
 
         chip_output.Log_info(':::::::::::::::::::Current Pointing Information::::::::::::::::::')
         chip_output.Log_info("RA: %f, DEC; %f" % (pointing.ra, pointing.dec))
@@ -68,7 +68,7 @@ class Observation(object):
             chip_output.Log_error("unrecognized PSF model type!!", flush=True)
 
         # Figure out shear fields
-        self.g1_field, self.g2_field, self.nshear = get_shear_field(config=self.config, shear_cat_file=shear_cat_file)
+        self.g1_field, self.g2_field, self.nshear = get_shear_field(config=self.config)
 
         # Apply astrometric simulation for pointing
         if self.config["obs_setting"]["enable_astrometric_model"]:
@@ -110,8 +110,11 @@ class Observation(object):
         if self.config["obs_setting"]["enable_straylight_model"]:
             filt.setFilterStrayLightPixel(jtime = pointing.jdt, sat_pos = np.array([pointing.sat_x, pointing.sat_y, pointing.sat_z]), pointing_radec = np.array([pointing.ra,pointing.dec]), sun_pos = np.array([pointing.sun_x,pointing.sun_y,pointing.sun_z]))
 
-        print("========================sky pix========================")
-        print(filt.sky_background)
+        chip_output.Log_info("========================sky pix========================")
+        chip_output.Log_info(filt.sky_background)
+
+        # [TODO] [TEST]
+        return
 
         if chip.survey_type == "photometric":
             sky_map = None
@@ -244,12 +247,6 @@ class Observation(object):
                         obj.g1, obj.g2 = 0., 0.
                     else:
                         obj.g1, obj.g2 = self.g1_field, self.g2_field
-                elif self.config["shear_setting"]["shear_type"] == "extra":
-                    try:
-                        # [TODO]: every object with individual shear from input catalog(s)
-                        obj.g1, obj.g2 = self.g1_field[j], self.g2_field[j]
-                    except:
-                        chip_output.Log_error("failed to load external shear.")
                 elif self.config["shear_setting"]["shear_type"] == "catalog":
                     pass
                 else:
@@ -411,7 +408,7 @@ class Observation(object):
 
         chip_output.Log_info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
 
-    def runExposure_MPI_PointingList(self, pointing_list, shear_cat_file=None, chips=None, use_mpi=False):
+    def runExposure_MPI_PointingList(self, pointing_list,chips=None, use_mpi=False):
         if use_mpi:
             comm = MPI.COMM_WORLD
             ind_thread = comm.Get_rank()

ObservationSim/_util.py

@@ -144,8 +144,8 @@ def makeSubDir_PointingList(path_dict, config, pointing_ID=0):
             pass
     return subImgdir, prefix
 
-def get_shear_field(config, shear_cat_file=None):
-    if not config["shear_setting"]["shear_type"] in ["constant", "extra", "catalog"]:
+def get_shear_field(config):
+    if not config["shear_setting"]["shear_type"] in ["constant", "catalog"]:
         raise ValueError("Please set a right 'shear_method' parameter.")
 
     if config["shear_setting"]["shear_type"] == "constant":
@@ -153,18 +153,6 @@ def get_shear_field(config, shear_cat_file=None):
         g2 = config["shear_setting"]["reduced_g2"]
         nshear = 1
         # TODO logging
-    elif config["shear_setting"]["shear_type"] == "extra":
-        # TODO logging
-        if not os.path.exists(shear_cat_file):
-            raise ValueError("Cannot find external shear catalog file.")
-        try:
-            shearCat = np.loadtxt(shear_cat_file)
-            nshear = shearCat.shape[0]
-            g1, g2 = shearCat[:, 0], shearCat[:, 1]
-        except:
-            print("Failed to read the shear catalog file.")
-            print("Setting to no shear.")
-            g1, g2 = 0., 0.
     else:
         g1, g2 = 0., 0.
         nshear = 0

config/config_C6.yaml

@@ -9,13 +9,13 @@
 # Base diretories and naming setup
 # Can add some of the command-line arguments here as well;
 # OK to pass either way or both, as long as they are consistent
-work_dir: "/share/home/fangyuedong/sim_v2/csst-simulation/workplace/"
+work_dir: "/share/home/fangyuedong/csst-simulation/workplace/"
 data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
-run_name: "C6_test_profiler"
+run_name: "C6_test"
 
 # Whether to use MPI
 run_option:
-  use_mpi: NO
+  use_mpi: YES
   # NOTE: "n_threads" paramters is currently not used in the backend
   # simulation codes. It should be implemented later in the web frontend
   # in order to config the number of threads to request from NAOC cluster
@@ -94,19 +94,19 @@ obs_setting:
   # - give a list of indexes of pointings: [ip_1, ip_2...]
   # - run all pointings: null
   # Note: only valid when a pointing list is specified
-  run_pointings: [0]
+  run_pointings: [0,1,2,3,4]
   
   # Run specific chip(s):
   # - give a list of indexes of chips: [ip_1, ip_2...]
   # - run all chips: null
   # Note: for all pointings
-  run_chips: [8]
+  run_chips: null
 
   # Whether to enable astrometric modeling
   enable_astrometric_model: True
 
   # Whether to enable straylight model
-  enable_straylight_model: False
+  enable_straylight_model: True
 
   # Cut by saturation magnitude in which band?
   cut_in_band: "z"
@@ -144,17 +144,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "catalog"
 
   # For constant shear filed
   reduced_g1: 0.
   reduced_g2: 0.
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################
@@ -162,25 +157,25 @@ ins_effects:
   # switches
   # Note: bias_16channel, gain_16channel, and shutter_effect
   # is currently not applicable to "FGS" observations
-  field_dist:     ON  # Whether to add field distortions
-  add_back:       ON  # Whether to add sky background
-  add_dark:       ON  # Whether to add dark noise
-  add_readout:    ON  # Whether to add read-out (Gaussian) noise
-  add_bias:       ON  # Whether to add bias-level to images
-  bias_16channel: ON  # Whether to add different biases for 16 channels
-  gain_16channel: ON  # Whether to make different gains for 16 channels
-  shutter_effect: ON  # Whether to add shutter effect
-  flat_fielding:  ON  # Whether to add flat-fielding effect
-  prnu_effect:    ON  # Whether to add PRNU effect
-  non_linear:     ON  # Whether to add non-linearity
-  cosmic_ray:     ON  # Whether to add cosmic-ray
-  cray_differ:    ON  # Whether to generate different cosmic ray maps CAL and MS output
-  cte_trail:      ON  # Whether to simulate CTE trails
-  saturbloom:     ON  # Whether to simulate Saturation & Blooming
-  add_badcolumns: ON  # Whether to add bad columns
-  add_hotpixels:  ON  # Whether to add hot pixels
-  add_deadpixels: ON  # Whether to add dead(dark) pixels
-  bright_fatter:  ON  # Whether to simulate Brighter-Fatter (also diffusion) effect
+  field_dist:     YES  # Whether to add field distortions
+  add_back:       YES  # Whether to add sky background
+  add_dark:       YES  # Whether to add dark noise
+  add_readout:    YES  # Whether to add read-out (Gaussian) noise
+  add_bias:       YES  # Whether to add bias-level to images
+  bias_16channel: YES  # Whether to add different biases for 16 channels
+  gain_16channel: YES  # Whether to make different gains for 16 channels
+  shutter_effect: YES  # Whether to add shutter effect
+  flat_fielding:  YES  # Whether to add flat-fielding effect
+  prnu_effect:    YES  # Whether to add PRNU effect
+  non_linear:     YES  # Whether to add non-linearity
+  cosmic_ray:     YES  # Whether to add cosmic-ray
+  cray_differ:    YES  # Whether to generate different cosmic ray maps CAL and MS output
+  cte_trail:      YES  # Whether to simulate CTE trails
+  saturbloom:     YES  # Whether to simulate Saturation & Blooming
+  add_badcolumns: YES  # Whether to add bad columns
+  add_hotpixels:  YES  # Whether to add hot pixels
+  add_deadpixels: YES  # Whether to add dead(dark) pixels
+  bright_fatter:  YES  # Whether to simulate Brighter-Fatter (also diffusion) effect
 
   # Values: 
   # default values have been defined individually for each chip in:

config/config_C6_fits.yaml

@@ -147,17 +147,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "catalog"
 
   # For constant shear filed
   reduced_g1: 0.
   reduced_g2: 0.
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/config_C6_test_wcs.yaml

@@ -141,17 +141,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "catalog"
 
   # For constant shear filed
   reduced_g1: 0.
   reduced_g2: 0.
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/config_NGP.yaml

@@ -138,17 +138,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "constant"
 
   # For constant shear filed
   reduced_g1: 0.026
   reduced_g2: 0.015
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/config_example.yaml

@@ -131,17 +131,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "constant"
 
   # For constant shear filed
   reduced_g1: 0.026
   reduced_g2: 0.015
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/config_fgs.yaml

@@ -143,17 +143,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "catalog"
 
   # For constant shear filed
   reduced_g1: 0.
   reduced_g2: 0.
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/config_testCASE.yaml

@@ -146,17 +146,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "catalog"
 
   # For constant shear filed
   reduced_g1: 0.
   reduced_g2: 0.
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

config/test_fd_C6.yaml

@@ -141,17 +141,12 @@ shear_setting:
   # Options to generate mock shear field:
   # "constant": all galaxies are assigned a constant reduced shear
   # "catalog": from catalog
-  # "extra": from seprate file
   shear_type: "catalog"
 
   # For constant shear filed
   reduced_g1: 0.
   reduced_g2: 0.
 
-  # Extra shear catalog
-  # (currently not used)
-  # shear_cat: "mockShear.cat"
-
 ###############################################
 # Instrumental effects setting
 ###############################################

run_C6.pbs

 #!/bin/bash
-
-#PBS -N SIMS
-#PBS -l nodes=wcl-2:ppn=40
-###PBS -l nodes=wcl-1:ppn=24+wcl-2:ppn=24+wcl-3:ppn=24+wcl-4:ppn=24+wcl-5:ppn=24+wcl-6:ppn=24
+#PBS -N C6_TEST
+#PBS -l walltime=70:00:00
+#PBS -j oe
+##PBS -l nodes=wcl-1:ppn=16+wcl-2:ppn=16
+#PBS -q batch
 #PBS -u fangyuedong
-###PBS -j oe
 
 cd $PBS_O_WORKDIR
 
-NP=40
+NP=48
+hostfile=wcl-1,wcl-2
 date
 
-mpirun -np $NP python3 /share/home/fangyuedong/sim_v2/csst-simulation/run_sim.py \
+mpirun --oversubscribe -H $hostfile -np $NP python /share/home/fangyuedong/csst-simulation/run_sim.py \
     --config_file config_C6.yaml \
     --catalog C6_Catalog \
-    -c /share/home/fangyuedong/sim_v2/csst-simulation/config
+    -c /share/home/fangyuedong/csst-simulation/config