fixed photometry/bad column issue, modify detector effects order

ObservationSim/Config/ChipOutput.py

@@ -37,8 +37,10 @@ class ChipOutput(object):
         fmt5 = "%10s %8.4f %8.4f %8.4f\n"
         self.hdr = hdr1 + hdr2 + hdr3 + hdr4 + hdr5
         self.fmt = fmt1 + fmt2 + fmt3 + fmt4 + fmt5
+        print("pointing_type = %s\n"%(pointing_type))
         if pointing_type == 'MS':
             self.cat = open(os.path.join(self.subdir, self.cat_name), "w")
+            print("Creating catalog file %s ...\n"%(os.path.join(self.subdir, self.cat_name)))
             self.cat.write(self.hdr)
 
     def updateHDR(self, hdr):

ObservationSim/Instrument/Chip/Effects.py

@@ -5,6 +5,7 @@ from numpy.core.fromnumeric import mean, size
 from numpy.random import Generator, PCG64
 import math
 from numba import jit
+from astropy import stats
 
 
 def AddOverscan(GSImage, overscan=1000, gain=1, widthl=27, widthr=27, widtht=8, widthb=8, read_noise=5):
@@ -31,7 +32,7 @@ def AddOverscan(GSImage, overscan=1000, gain=1, widthl=27, widthr=27, widtht=8,
     return newimg
 
 
-def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed=20210304, biaslevel=500):
+def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed=20210304, biaslevel=0):
     # Also called bad pixels, including hot pixels and dead pixels
     # Hot Pixel > 20e-/s
     # Dead Pixel < 70%*Mean
@@ -68,10 +69,11 @@ def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed=
     return GSImage
 
 
-def BadColumns(GSImage, seed=20210309, chipid=1):
+def BadColumns(GSImage, seed=20240309, chipid=1):
     # Set bad column values
     ysize,xsize = GSImage.array.shape
     subarr = GSImage.array[int(ysize*0.1):int(ysize*0.12), int(xsize*0.1):int(xsize*0.12)]
+    subarr = stats.sigma_clip(subarr, sigma=4, cenfunc='median', maxiters=3, masked=False)
     meanimg = np.median(subarr)
     stdimg = np.std(subarr)
     seed += chipid
@@ -80,15 +82,19 @@ def BadColumns(GSImage, seed=20210309, chipid=1):
     rgxpos = Generator(PCG64(int(seed*1.2)))
     rgdn = Generator(PCG64(int(seed*1.3)))
 
-    nbadsecA,nbadsecD = rgn.integers(low=1, high=5, size=2) 
+    nbadsecA,nbadsecD = rgn.integers(low=1, high=5, size=2)
     collen = rgcollen.integers(low=int(ysize*0.1), high=int(ysize*0.7), size=(nbadsecA+nbadsecD)) 
     xposit = rgxpos.integers(low=int(xsize*0.05), high=int(xsize*0.95), size=(nbadsecA+nbadsecD))
-    signs = 2*rgdn.integers(0,2,size=(nbadsecA+nbadsecD))-1
-    dn = rgdn.integers(low=meanimg*0.4, high=meanimg*0.8, size=(nbadsecA+nbadsecD))*signs
+    print(xposit+1)
+    # signs = 2*rgdn.integers(0,2,size=(nbadsecA+nbadsecD))-1
+    if meanimg>0:
+        dn = rgdn.integers(low=meanimg*1.3+50, high=meanimg*2+150, size=(nbadsecA+nbadsecD)) #*signs
+    elif meanimg<0:
+        dn = rgdn.integers(low=meanimg*2-150, high=meanimg*1.3-50, size=(nbadsecA+nbadsecD)) #*signs
     for badcoli in range(nbadsecA):
-        GSImage.array[(ysize-collen[badcoli]):ysize,xposit[badcoli]:(xposit[badcoli]+1)] += np.random.random((collen[badcoli],1))*stdimg*3+dn[badcoli]
+        GSImage.array[(ysize-collen[badcoli]):ysize,xposit[badcoli]:(xposit[badcoli]+1)] = (np.abs(np.random.normal(0, stdimg*2, (collen[badcoli],1)))+dn[badcoli])
     for badcoli in range(nbadsecD):
-        GSImage.array[0:collen[badcoli+nbadsecA],xposit[badcoli+nbadsecA]:(xposit[badcoli+nbadsecA]+1)] += np.random.random((collen[badcoli+nbadsecA],1))*stdimg*3+dn[badcoli+nbadsecA]
+        GSImage.array[0:collen[badcoli+nbadsecA],xposit[badcoli+nbadsecA]:(xposit[badcoli+nbadsecA]+1)] = (np.abs(np.random.normal(0, stdimg*2, (collen[badcoli+nbadsecA],1)))+dn[badcoli+nbadsecA])
     return GSImage
 
 
@@ -223,7 +229,7 @@ def PRNU_Img(xsize, ysize, sigma=0.01, seed=202101):
     return prnuimg
 
 
-def NonLinearity(GSImage, beta1=1E-7, beta2=1E-10):
+def NonLinearity(GSImage, beta1=5E-7, beta2=0):
     NonLinear_f = lambda x, beta_1, beta_2: x - beta_1*x*x + beta_2*x*x*x
     GSImage.applyNonlinearity(NonLinear_f, beta1, beta2)
     return GSImage

ObservationSim/Instrument/Filter.py

@@ -73,5 +73,5 @@ class Filter(object):
     def getPhotonE(self):
         return photonEnergy(self.effective_wavelength)
 
-    def getSkyNoise(self, exptime, gain):
+    def getSkyNoise(self, exptime, gain=1.):
         return self.sky_background * exptime / gain

ObservationSim/ObservationSim.py

@@ -97,6 +97,7 @@ class Observation(object):
                 # Load SED
                 if obj.type == 'star':
                     normF = chip.normF_star
+                    #continue
                     try:
                         obj.load_SED(
                             survey_type=chip.survey_type, 
@@ -108,6 +109,7 @@ class Observation(object):
                         continue
                 elif obj.type == 'galaxy': # or obj.type == quasar
                     normF = chip.normF_galaxy
+                    # continue
                     obj.load_SED(
                         sed_path=sed_dir, 
                         survey_type=chip.survey_type, 
@@ -116,6 +118,7 @@ class Observation(object):
                         target_filt=filt)
                 elif obj.type == 'quasar':
                     normF = chip.normF_galaxy
+                    # continue
                     obj.load_SED(
                         sed_path=sed_dir, 
                         survey_type=chip.survey_type, 
@@ -200,7 +203,7 @@ class Observation(object):
 
         # Detector Effects
         # ===========================================================
-        chip.img = chip.addNoise(config=self.config, tel=self.tel, filt=filt, img=chip.img, sky_map=sky_map)
+        # chip.img = chip.addNoise(config=self.config, tel=self.tel, filt=filt, img=chip.img, sky_map=sky_map)
         # chip.img = chip.addEffects(config=self.config, img=chip.img, chip_output=chip_output, filt=filt, pointing_ID=pointing_ID)
 
         # whether to output zero, dark, flat calibration images.
@@ -214,7 +217,8 @@ class Observation(object):
             img_rot=img_rot,
             pointing_ID=pointing_ID,
             timestamp_obs=timestamp_obs,
-            pointing_type=pointing_type)
+            pointing_type=pointing_type,
+            sky_map=sky_map)
         
         if pointing_type == 'MS':
             datetime_obs = datetime.fromtimestamp(timestamp_obs)
@@ -309,13 +313,20 @@ class Observation(object):
                 sed_dir=self.path_dict["SED_dir"])
             print("finished running chip#%d..."%(chip.chipID), flush=True)
 
-    def runExposure_MPI_PointingList(self, ra_cen=None, dec_cen=None, pRange=None, timestamp_obs=[1621915200], pointing_type=['MS'], img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None):
+    def runExposure_MPI_PointingList(self, ra_cen=None, dec_cen=None, pRange=None, timestamp_obs=np.array([1621915200]), pointing_type=np.array(['MS']), img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None):
         comm = MPI.COMM_WORLD
         ind_thread = comm.Get_rank()
         num_thread = comm.Get_size()
 
         nchips_per_fp = len(self.chip_list)
 
+        # TEMP
+        if len(timestamp_obs) == 1:
+            timestamp_obs = np.tile(timestamp_obs, len(ra_cen))
+            pointing_type = np.tile(pointing_type, len(ra_cen))
+        
+        timestamp_obs = timestamp_obs[pRange]
+        pointing_type = pointing_type[pRange]
         ra_cen = ra_cen[pRange]
         dec_cen = dec_cen[pRange]
 
@@ -325,11 +336,6 @@ class Observation(object):
         else:
             pStart = 0
 
-        # TEMP
-        if len(timestamp_obs) == 1:
-            timestamp_obs *= len(pRange)
-            pointing_type *= len(pRange)
-
         for ipoint in range(len(ra_cen)):
             for ichip in range(nchips_per_fp):
                 i = ipoint*nchips_per_fp + ichip

ObservationSim/PSF/PSFInterp/PSFInterp.py

@@ -171,8 +171,8 @@ class PSFInterp(PSFModel):
             img = galsim.ImageF(imPSF, scale=pixSize)
             gsp = galsim.GSParams(folding_threshold=folding_threshold)
             self.psf = galsim.InterpolatedImage(img, gsparams=gsp)
-            gaussPSF = galsim.Gaussian(sigma=0.04, gsparams=gsp)
-            self.psf    = galsim.Convolve(gaussPSF, self.psf)
+            # gaussPSF = galsim.Gaussian(sigma=0.04, gsparams=gsp)
+            # self.psf    = galsim.Convolve(gaussPSF, self.psf)
             return self.PSFspin(x=px/0.01, y=py/0.01)
         return imPSF
 

ObservationSim/Pointing.py

@@ -9,6 +9,8 @@ data_dir = "/data/simudata/CSSOSDataProductsSims/data/"
 config_file = os.path.join(work_dir, "ObservationSim.cfg")
 config = ReadConfig(config_file)
 
+exp_time = float(config['exp_time'])
+
 # Read Pointing list
 pointing_file = os.path.join(data_dir, "pointing10_20210202.dat")
 f = open(pointing_file, 'r')
@@ -37,12 +39,16 @@ pointing_type = ['CAL']*ncal + pointing_type
 t0 = datetime(2021, 5, 25, 12, 0, 0)
 t = datetime.timestamp(t0)
 timestamp_obs = []
+delta_t = 10 # Time elapsed between exposures (minutes)
 for i in range(len(pointing_type)):
     timestamp_obs.append(t)
     if pointing_type[i] == 'CAL':
-        t += 3 * 5 * 60 # 3 calibration exposure
+        t += 3 * delta_t * 60 # 3 calibration exposure
     elif pointing_type[i] == 'MS':
-        t += 5 * 60
+        t += delta_t * 60
+
+timestamp_obs = np.array(timestamp_obs)
+pointing_type = np.array(pointing_type)
 
 # Define the range of pointing list
 pRange = range(0, 2)
\ No newline at end of file

ObservationSim/run.pbs

@@ -7,7 +7,7 @@
 ##mpdboot -n 10 -f  ./mpd.hosts
 
 ##PBS -j oe
-#PBS -l nodes=comput110:ppn=80
+#PBS -l nodes=comput108:ppn=80
 #####PBS -q longq
 #PBS -q batch
 #PBS -u fangyuedong

ObservationSim/runExposure.py

@@ -20,4 +20,4 @@ gc.enable()
 # Testing run pointing list (using MPI)
 obs = Observation(work_dir=work_dir, data_dir=data_dir)
 # obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type)
-obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=150.)
+obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=exp_time)

test/ObservationSim.cfg

@@ -7,6 +7,7 @@
 # Observation setting
 date_obs      210525                  # Observation date [yymmdd]
 time_obs      120000                  # Observation time [hhmmss]
+exp_time      150.                    # Exposure time [seconds]
 ra_center     60.0                    # Telesscope pointing center [degree] (Default)
 dec_center   -40.0                     
 psf_rcont     0.15,0.8                # Radius of the 80% flux concentration (for a Gaussian PSF)
@@ -18,7 +19,7 @@ survey_type   Photometric             # Survey simulation option:
                                       # 'Photometric': only run sims for photometric chips
                                       # 'Spectroscopic': only run sims for spectroscopic chps
                                       # 'All': run sims for all chips
-psf_model     Interp                  # which PSF model to use:
+psf_model     Interp                  # Which PSF model to use:
                                       # 'Gauss': simple gaussin profile
                                       # 'Interp': Interpolated PSF from measured data 
 
@@ -26,8 +27,9 @@ psf_model     Interp                  # which PSF model to use:
 # Instrument effects
 #===========================================================
 field_dist       y                    # Y/N: Whether to add field distortion
-abs_back         y                    # Y: add sky background + sky noise & dark + dark noise 
-                                      # N: only add sky noise and dark noise
+add_back         y                    # Y/N: Whether to add sky background
+add_dark         y                    # Y/N: Whether to add dark noise
+add_readout      y                    # Y/N: Whether to add read-out (Gaussian) noise
 add_bias         y                    # Y/N: Whether to add bias-level to image
 shutter_effect   y                    # Y/N: Whether to apply shutter effect
 shutter_output   n                    # Y/N: Whether to export shutter effect 16-bit image (<=65535)
@@ -56,6 +58,7 @@ NDark            1                    # Number of frames to be exported
 NFlat            1                    # Number of frames to be exported
 dark_exptime     300                  # Exposure time for dark current frame
 flat_exptime     150                  # Exposure time for flat field frame
+readout_time     40                   # Time for read-out by each channel
 
 #===========================================================
 # Shear method
@@ -85,6 +88,7 @@ seed_mock            12345678         # Seed for generating random values of ra,
 seed_star            121212121        # Seed for generating random redshift, sed_type for stars
 seed_gal             212121212        # Seed for generating random redshift, sed_type for galaxies
 seed_Av              121212           # Seed for generating random intrinsic extinction
+seed_poisson         20210601         # Seed for Poisson noise
 seed_CR              20210317         # Seed for generating random cosmic ray map
 seed_flat            20210101         # Seed for generating random flat field
 seed_prnu            20210102         # Seed for photo-response non-uniformity
@@ -93,4 +97,5 @@ seed_gainNonUniform  20210202         # Seed for Gain nonuniformity
 seed_biasNonUniform  20210203         # Seed for Bias nonuniformity
 seed_rnNonUniform    20210204         # Seed for ReadNoise nonuniformity
 seed_badcolumns      20240309         # Initial Seed for Bad Columns
-seed_defective       20210304         # Seed for Defective(bad) pixels
\ No newline at end of file
+seed_defective       20210304         # Seed for Defective(bad) pixels
+seed_readout         20210601         # Seed for Read-out gaussian noise
\ No newline at end of file