From 5cf3f5bbe1da680e9a669125ca312f3f712ec0cc Mon Sep 17 00:00:00 2001 From: fangyuedong Date: Wed, 5 Jul 2023 08:27:15 +0800 Subject: [PATCH] modify codes to test straylight model crashing issue --- ObservationSim/ObservationSim.py | 19 +++++------ ObservationSim/_util.py | 16 ++-------- config/config_C6.yaml | 55 +++++++++++++++----------------- config/config_C6_fits.yaml | 5 --- config/config_C6_test_wcs.yaml | 5 --- config/config_NGP.yaml | 5 --- config/config_example.yaml | 5 --- config/config_fgs.yaml | 5 --- config/config_testCASE.yaml | 5 --- config/test_fd_C6.yaml | 5 --- run_C6.pbs | 17 +++++----- 11 files changed, 44 insertions(+), 98 deletions(-) diff --git a/ObservationSim/ObservationSim.py b/ObservationSim/ObservationSim.py index 14bc8b4..094be9d 100755 --- a/ObservationSim/ObservationSim.py +++ b/ObservationSim/ObservationSim.py @@ -48,7 +48,7 @@ class Observation(object): self.filter_list.append(filt) self.all_filter.append(filt) - def run_one_chip(self, chip, filt, pointing, chip_output, wcs_fp=None, psf_model=None, shear_cat_file=None, cat_dir=None, sed_dir=None): + def run_one_chip(self, chip, filt, pointing, chip_output, wcs_fp=None, psf_model=None, cat_dir=None, sed_dir=None): chip_output.Log_info(':::::::::::::::::::Current Pointing Information::::::::::::::::::') chip_output.Log_info("RA: %f, DEC; %f" % (pointing.ra, pointing.dec)) @@ -68,7 +68,7 @@ class Observation(object): chip_output.Log_error("unrecognized PSF model type!!", flush=True) # Figure out shear fields - self.g1_field, self.g2_field, self.nshear = get_shear_field(config=self.config, shear_cat_file=shear_cat_file) + self.g1_field, self.g2_field, self.nshear = get_shear_field(config=self.config) # Apply astrometric simulation for pointing if self.config["obs_setting"]["enable_astrometric_model"]: @@ -110,8 +110,11 @@ class Observation(object): if self.config["obs_setting"]["enable_straylight_model"]: filt.setFilterStrayLightPixel(jtime = pointing.jdt, sat_pos = np.array([pointing.sat_x, pointing.sat_y, pointing.sat_z]), pointing_radec = np.array([pointing.ra,pointing.dec]), sun_pos = np.array([pointing.sun_x,pointing.sun_y,pointing.sun_z])) - print("========================sky pix========================") - print(filt.sky_background) + chip_output.Log_info("========================sky pix========================") + chip_output.Log_info(filt.sky_background) + + # [TODO] [TEST] + return if chip.survey_type == "photometric": sky_map = None @@ -244,12 +247,6 @@ class Observation(object): obj.g1, obj.g2 = 0., 0. else: obj.g1, obj.g2 = self.g1_field, self.g2_field - elif self.config["shear_setting"]["shear_type"] == "extra": - try: - # [TODO]: every object with individual shear from input catalog(s) - obj.g1, obj.g2 = self.g1_field[j], self.g2_field[j] - except: - chip_output.Log_error("failed to load external shear.") elif self.config["shear_setting"]["shear_type"] == "catalog": pass else: @@ -411,7 +408,7 @@ class Observation(object): chip_output.Log_info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) )) - def runExposure_MPI_PointingList(self, pointing_list, shear_cat_file=None, chips=None, use_mpi=False): + def runExposure_MPI_PointingList(self, pointing_list,chips=None, use_mpi=False): if use_mpi: comm = MPI.COMM_WORLD ind_thread = comm.Get_rank() diff --git a/ObservationSim/_util.py b/ObservationSim/_util.py index 6c5af29..9f12f5d 100755 --- a/ObservationSim/_util.py +++ b/ObservationSim/_util.py @@ -144,8 +144,8 @@ def makeSubDir_PointingList(path_dict, config, pointing_ID=0): pass return subImgdir, prefix -def get_shear_field(config, shear_cat_file=None): - if not config["shear_setting"]["shear_type"] in ["constant", "extra", "catalog"]: +def get_shear_field(config): + if not config["shear_setting"]["shear_type"] in ["constant", "catalog"]: raise ValueError("Please set a right 'shear_method' parameter.") if config["shear_setting"]["shear_type"] == "constant": @@ -153,18 +153,6 @@ def get_shear_field(config, shear_cat_file=None): g2 = config["shear_setting"]["reduced_g2"] nshear = 1 # TODO logging - elif config["shear_setting"]["shear_type"] == "extra": - # TODO logging - if not os.path.exists(shear_cat_file): - raise ValueError("Cannot find external shear catalog file.") - try: - shearCat = np.loadtxt(shear_cat_file) - nshear = shearCat.shape[0] - g1, g2 = shearCat[:, 0], shearCat[:, 1] - except: - print("Failed to read the shear catalog file.") - print("Setting to no shear.") - g1, g2 = 0., 0. else: g1, g2 = 0., 0. nshear = 0 diff --git a/config/config_C6.yaml b/config/config_C6.yaml index b68442f..14aa0db 100644 --- a/config/config_C6.yaml +++ b/config/config_C6.yaml @@ -9,13 +9,13 @@ # Base diretories and naming setup # Can add some of the command-line arguments here as well; # OK to pass either way or both, as long as they are consistent -work_dir: "/share/home/fangyuedong/sim_v2/csst-simulation/workplace/" +work_dir: "/share/home/fangyuedong/csst-simulation/workplace/" data_dir: "/share/simudata/CSSOSDataProductsSims/data/" -run_name: "C6_test_profiler" +run_name: "C6_test" # Whether to use MPI run_option: - use_mpi: NO + use_mpi: YES # NOTE: "n_threads" paramters is currently not used in the backend # simulation codes. It should be implemented later in the web frontend # in order to config the number of threads to request from NAOC cluster @@ -94,19 +94,19 @@ obs_setting: # - give a list of indexes of pointings: [ip_1, ip_2...] # - run all pointings: null # Note: only valid when a pointing list is specified - run_pointings: [0] + run_pointings: [0,1,2,3,4] # Run specific chip(s): # - give a list of indexes of chips: [ip_1, ip_2...] # - run all chips: null # Note: for all pointings - run_chips: [8] + run_chips: null # Whether to enable astrometric modeling enable_astrometric_model: True # Whether to enable straylight model - enable_straylight_model: False + enable_straylight_model: True # Cut by saturation magnitude in which band? cut_in_band: "z" @@ -144,17 +144,12 @@ shear_setting: # Options to generate mock shear field: # "constant": all galaxies are assigned a constant reduced shear # "catalog": from catalog - # "extra": from seprate file shear_type: "catalog" # For constant shear filed reduced_g1: 0. reduced_g2: 0. - # Extra shear catalog - # (currently not used) - # shear_cat: "mockShear.cat" - ############################################### # Instrumental effects setting ############################################### @@ -162,25 +157,25 @@ ins_effects: # switches # Note: bias_16channel, gain_16channel, and shutter_effect # is currently not applicable to "FGS" observations - field_dist: ON # Whether to add field distortions - add_back: ON # Whether to add sky background - add_dark: ON # Whether to add dark noise - add_readout: ON # Whether to add read-out (Gaussian) noise - add_bias: ON # Whether to add bias-level to images - bias_16channel: ON # Whether to add different biases for 16 channels - gain_16channel: ON # Whether to make different gains for 16 channels - shutter_effect: ON # Whether to add shutter effect - flat_fielding: ON # Whether to add flat-fielding effect - prnu_effect: ON # Whether to add PRNU effect - non_linear: ON # Whether to add non-linearity - cosmic_ray: ON # Whether to add cosmic-ray - cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output - cte_trail: ON # Whether to simulate CTE trails - saturbloom: ON # Whether to simulate Saturation & Blooming - add_badcolumns: ON # Whether to add bad columns - add_hotpixels: ON # Whether to add hot pixels - add_deadpixels: ON # Whether to add dead(dark) pixels - bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect + field_dist: YES # Whether to add field distortions + add_back: YES # Whether to add sky background + add_dark: YES # Whether to add dark noise + add_readout: YES # Whether to add read-out (Gaussian) noise + add_bias: YES # Whether to add bias-level to images + bias_16channel: YES # Whether to add different biases for 16 channels + gain_16channel: YES # Whether to make different gains for 16 channels + shutter_effect: YES # Whether to add shutter effect + flat_fielding: YES # Whether to add flat-fielding effect + prnu_effect: YES # Whether to add PRNU effect + non_linear: YES # Whether to add non-linearity + cosmic_ray: YES # Whether to add cosmic-ray + cray_differ: YES # Whether to generate different cosmic ray maps CAL and MS output + cte_trail: YES # Whether to simulate CTE trails + saturbloom: YES # Whether to simulate Saturation & Blooming + add_badcolumns: YES # Whether to add bad columns + add_hotpixels: YES # Whether to add hot pixels + add_deadpixels: YES # Whether to add dead(dark) pixels + bright_fatter: YES # Whether to simulate Brighter-Fatter (also diffusion) effect # Values: # default values have been defined individually for each chip in: diff --git a/config/config_C6_fits.yaml b/config/config_C6_fits.yaml index 4837152..074a627 100644 --- a/config/config_C6_fits.yaml +++ b/config/config_C6_fits.yaml @@ -147,17 +147,12 @@ shear_setting: # Options to generate mock shear field: # "constant": all galaxies are assigned a constant reduced shear # "catalog": from catalog - # "extra": from seprate file shear_type: "catalog" # For constant shear filed reduced_g1: 0. reduced_g2: 0. - # Extra shear catalog - # (currently not used) - # shear_cat: "mockShear.cat" - ############################################### # Instrumental effects setting ############################################### diff --git a/config/config_C6_test_wcs.yaml b/config/config_C6_test_wcs.yaml index ad62334..7731972 100644 --- a/config/config_C6_test_wcs.yaml +++ b/config/config_C6_test_wcs.yaml @@ -141,17 +141,12 @@ shear_setting: # Options to generate mock shear field: # "constant": all galaxies are assigned a constant reduced shear # "catalog": from catalog - # "extra": from seprate file shear_type: "catalog" # For constant shear filed reduced_g1: 0. reduced_g2: 0. - # Extra shear catalog - # (currently not used) - # shear_cat: "mockShear.cat" - ############################################### # Instrumental effects setting ############################################### diff --git a/config/config_NGP.yaml b/config/config_NGP.yaml index 37aab5a..2a986b3 100644 --- a/config/config_NGP.yaml +++ b/config/config_NGP.yaml @@ -138,17 +138,12 @@ shear_setting: # Options to generate mock shear field: # "constant": all galaxies are assigned a constant reduced shear # "catalog": from catalog - # "extra": from seprate file shear_type: "constant" # For constant shear filed reduced_g1: 0.026 reduced_g2: 0.015 - # Extra shear catalog - # (currently not used) - # shear_cat: "mockShear.cat" - ############################################### # Instrumental effects setting ############################################### diff --git a/config/config_example.yaml b/config/config_example.yaml index a3cc402..5dcec7b 100644 --- a/config/config_example.yaml +++ b/config/config_example.yaml @@ -131,17 +131,12 @@ shear_setting: # Options to generate mock shear field: # "constant": all galaxies are assigned a constant reduced shear # "catalog": from catalog - # "extra": from seprate file shear_type: "constant" # For constant shear filed reduced_g1: 0.026 reduced_g2: 0.015 - # Extra shear catalog - # (currently not used) - # shear_cat: "mockShear.cat" - ############################################### # Instrumental effects setting ############################################### diff --git a/config/config_fgs.yaml b/config/config_fgs.yaml index 00bca3b..3a0d021 100644 --- a/config/config_fgs.yaml +++ b/config/config_fgs.yaml @@ -143,17 +143,12 @@ shear_setting: # Options to generate mock shear field: # "constant": all galaxies are assigned a constant reduced shear # "catalog": from catalog - # "extra": from seprate file shear_type: "catalog" # For constant shear filed reduced_g1: 0. reduced_g2: 0. - # Extra shear catalog - # (currently not used) - # shear_cat: "mockShear.cat" - ############################################### # Instrumental effects setting ############################################### diff --git a/config/config_testCASE.yaml b/config/config_testCASE.yaml index 062853b..503d753 100644 --- a/config/config_testCASE.yaml +++ b/config/config_testCASE.yaml @@ -146,17 +146,12 @@ shear_setting: # Options to generate mock shear field: # "constant": all galaxies are assigned a constant reduced shear # "catalog": from catalog - # "extra": from seprate file shear_type: "catalog" # For constant shear filed reduced_g1: 0. reduced_g2: 0. - # Extra shear catalog - # (currently not used) - # shear_cat: "mockShear.cat" - ############################################### # Instrumental effects setting ############################################### diff --git a/config/test_fd_C6.yaml b/config/test_fd_C6.yaml index 9e720fb..6f85d4e 100644 --- a/config/test_fd_C6.yaml +++ b/config/test_fd_C6.yaml @@ -141,17 +141,12 @@ shear_setting: # Options to generate mock shear field: # "constant": all galaxies are assigned a constant reduced shear # "catalog": from catalog - # "extra": from seprate file shear_type: "catalog" # For constant shear filed reduced_g1: 0. reduced_g2: 0. - # Extra shear catalog - # (currently not used) - # shear_cat: "mockShear.cat" - ############################################### # Instrumental effects setting ############################################### diff --git a/run_C6.pbs b/run_C6.pbs index dca7a27..6c7837a 100755 --- a/run_C6.pbs +++ b/run_C6.pbs @@ -1,18 +1,19 @@ #!/bin/bash - -#PBS -N SIMS -#PBS -l nodes=wcl-2:ppn=40 -###PBS -l nodes=wcl-1:ppn=24+wcl-2:ppn=24+wcl-3:ppn=24+wcl-4:ppn=24+wcl-5:ppn=24+wcl-6:ppn=24 +#PBS -N C6_TEST +#PBS -l walltime=70:00:00 +#PBS -j oe +##PBS -l nodes=wcl-1:ppn=16+wcl-2:ppn=16 +#PBS -q batch #PBS -u fangyuedong -###PBS -j oe cd $PBS_O_WORKDIR -NP=40 +NP=48 +hostfile=wcl-1,wcl-2 date -mpirun -np $NP python3 /share/home/fangyuedong/sim_v2/csst-simulation/run_sim.py \ +mpirun --oversubscribe -H $hostfile -np $NP python /share/home/fangyuedong/csst-simulation/run_sim.py \ --config_file config_C6.yaml \ --catalog C6_Catalog \ - -c /share/home/fangyuedong/sim_v2/csst-simulation/config + -c /share/home/fangyuedong/csst-simulation/config -- GitLab