Merge branch 'sim_scheduler' of...

Merge branch 'sim_scheduler' of https://csst-tb.bao.ac.cn/code/csst_sim/csst-simulation into sim_scheduler

Conflicts:
	config/obs_config_SCI_WIDE_phot.yaml

ObservationSim/MockObject/FlatLED.py

@@ -35,12 +35,21 @@ cwaves_fwhm = {'LED1': 110, 'LED2': 120, 'LED3': 200, 'LED4': 300, 'LED5': 300,
           'LED8': 260, 'LED9': 400, 'LED10': 370, 'LED11': 500, 'LED12': 1400, 'LED13': 90, 'LED14': 100}
 # LED_QE = {'LED1': 0.3, 'LED2': 0.4, 'LED13': 0.5, 'LED14': 0.5, 'LED10': 0.4}
 # e-/ms
-fluxLED = {'LED1': 0.16478729, 'LED2': 0.084220931, 'LED3': 2.263360617, 'LED4': 2.190623489, 'LED5': 0.703504768,
-           'LED6': 0.446117963, 'LED7': 0.647122098, 'LED8': 0.922313442,
-           'LED9': 0.987278143, 'LED10': 2.043989167, 'LED11': 0.612571429, 'LED12': 1.228915663, 'LED13': 0.17029384,
-           'LED14': 0.27842925}
+# fluxLED = {'LED1': 0.16478729, 'LED2': 0.084220931, 'LED3': 2.263360617, 'LED4': 2.190623489, 'LED5': 0.703504768,
+#            'LED6': 0.446117963, 'LED7': 0.647122098, 'LED8': 0.922313442,
+#            'LED9': 0.987278143, 'LED10': 2.043989167, 'LED11': 0.612571429, 'LED12': 1.228915663, 'LED13': 0.17029384,
+#            'LED14': 0.27842925}
+
+# e-/ms
+fluxLED = {'LED1': 15, 'LED2': 15, 'LED3': 12.5, 'LED4': 9, 'LED5': 9,
+           'LED6': 9, 'LED7': 9, 'LED8': 9, 'LED9': 9, 'LED10': 12.5, 'LED11': 15, 'LED12':15, 'LED13': 12.5,
+           'LED14': 12.5}
+# fluxLEDL = {'LED1': 10, 'LED2': 10, 'LED3': 10, 'LED4': 10, 'LED5': 10,
+#            'LED6': 10, 'LED7': 10, 'LED8': 10, 'LED9': 10, 'LED10': 10, 'LED11': 10, 'LED12':10, 'LED13': 10,
+#            'LED14': 10}
 
 mirro_eff = {'GU':0.61, 'GV':0.8, 'GI':0.8}
+# mirro_eff = {'GU':1, 'GV':1, 'GI':1}
 
 class FlatLED(object):
     def __init__(self, chip,filt, flatDir = None, logger=None):
@@ -115,7 +124,7 @@ class FlatLED(object):
             t_spec = np.trapz(f_spec*ccd_eff*fil_eff, w_list)
             # print(i, np.mean(unitFlatImg), t_spec, exp_t)
             unitFlatImg = unitFlatImg * t_spec
-
+            # print("DEBUG1:---------------",np.mean(unitFlatImg))
             ledFlat = ledFlat+unitFlatImg*exp_t
         return ledFlat
 
@@ -134,6 +143,7 @@ class FlatLED(object):
             led_wave = cwaves[led_type]
             led_fwhm = cwaves_fwhm[led_type]
             led_spec = self.gaussian1d_profile_led(led_wave, led_fwhm)
+            # print("DEBUG1:---------------",np.mean(ledFlat_))
             ledspec_map = self.calculateLEDSpec(
                 skyMap=ledFlat_,
                 blueLimit=self.filt.blue_limit,
@@ -160,11 +170,15 @@ class FlatLED(object):
         xlist = np.arange(xc-x_radii, xc+x_radii, 0.5)
         xlist_ = np.zeros(len(xlist) + 2)
         xlist_[1:-1] = xlist
-        xlist_[0] = 2550
-        xlist_[-1] = 10000
+        xlist_[0] = 2000
+        xlist_[-1] = 18000
+        ids1 = xlist>xc-fwhm
+        ids2 = xlist[ids1]<xc+fwhm
         data = np.exp((-(xlist-xc)*(xlist-xc))/(2*sigma*sigma))/(np.sqrt(2*math.pi)*sigma)
+        scale = 1/np.trapz(data[ids1][ids2], xlist[ids1][ids2])
         data_ = np.zeros(len(xlist) + 2)
-        data_[1:-1] = data
+        data_[1:-1] = data*scale
+        # print("DEBUG:-------------------------------",np.sum(data_), scale)
         return Table(np.array([xlist_.astype(np.float32), data_.astype(np.float32)]).T, names=('WAVELENGTH', 'FLUX'))
 
     def calculateLEDSpec(self, skyMap=None, blueLimit=4200, redLimit=6500,
@@ -232,7 +246,7 @@ class FlatLED(object):
                                         tar_spec=spec,
                                         band_start=tbstart, band_end=tbend,
                                         conf=conf2,
-                                        flat_cube=flat_cube, ignoreBeam=['D', 'E'])
+                                        flat_cube=flat_cube)
 
                     spec_orders = sdp.compute_spec_orders()
 
@@ -356,7 +370,7 @@ class FlatLED(object):
         else:
             fimg = fImg.array
 
-        fimg = fimg * pixelSize * pixelSize
+        # fimg = fimg * pixelSize * pixelSize
 
         return fimg
 

ObservationSim/sim_steps/add_LED_flat.py

@@ -13,8 +13,15 @@ def add_LED_Flat(self, chip, filt, tel, pointing, catalog, obs_param):
             print("LED OPEN--------")
 
             led_obj = FlatLED(chip, filt)
-
             led_flat = led_obj.drawObj_LEDFlat(led_type_list=obs_param["LED_TYPE"], exp_t_list=obs_param["LED_TIME"])
             pf_map = led_flat
-    chip.img = chip.img + led_flat
+
+    if obs_param["shutter_effect"] == True:
+        pf_map = pf_map * chip.shutter_img
+        pf_map = np.array(pf_map, dtype='float32')
+        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True])
+    else:
+        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [False,'','','',''])
+
+    chip.img = chip.img + pf_map
     return chip, filt, tel, pointing
\ No newline at end of file

ObservationSim/sim_steps/readout_output.py

@@ -16,7 +16,7 @@ def add_prescan_overscan(self, chip, filt, tel, pointing, catalog, obs_param):
     if obs_param["add_dark"] == True:
         ny = int(chip.npix_y/2)
         base_dark = (ny-1)*(chip.readout_time/ny)*chip.dark_noise
-        chip.img.array[(chip.prescan_y+ny):-(chip.prescan_y+ny),:] = bese_dark
+        chip.img.array[(chip.prescan_y+ny):-(chip.prescan_y+ny),:] = base_dark
     return chip, filt, tel, pointing
 
 def add_readout_noise(self, chip, filt, tel, pointing, catalog, obs_param):
@@ -43,6 +43,12 @@ def quantization_and_output(self, chip, filt, tel, pointing, catalog, obs_param)
     if not hasattr(self, 'h_ext'):
         _, _ = self.prepare_headers(chip=chip, pointing=pointing)
         self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [False,'','','',''])
+    
+    gains1 = list(chip.gain_channel[0:8])
+    gains2 = list(chip.gain_channel[8:])
+    gains2.reverse()
+    gains = np.append(gains1,gains2)
+    self.updateHeaderInfo(header_flag='ext', keys = ['GAIN01','GAIN02','GAIN03','GAIN04','GAIN05','GAIN06','GAIN07','GAIN08','GAIN09','GAIN10','GAIN11','GAIN12','GAIN13','GAIN14','GAIN15','GAIN16'], values = gains)
 
     if obs_param["format_output"] == True:
         self.chip_output.Log_info("  Apply 1*16 format")

config/obs_config_Calibration_BIAS.yaml

+---
+###############################################
+#
+#  Configuration file for CSST simulation
+#  For single exposure type:
+#                BIAS
+#  CSST-Sim Group, 2024/01/08
+#
+###############################################
+
+# Observation type
+obs_type: "BIAS"
+obs_type_code: "120"
+obs_id: "00000001"
+# Define list of chips
+run_chips: [17]
+
+# Define observation sequence
+call_sequence:
+#   # Accumulate fluxes from objects
+#   led_calib_model:
+#     #"LED": ['LED1','LED2','LED3','LED4','LED5','LED6','LED7','LED8','LED9','LED10','LED11','LED12','LED13','LED14'] or null
+#     #'LED1': '275', 'LED2': '310', 'LED3': '430', 'LED4': '505', 'LED5': '525', 'LED6': '590', 'LED7': '670',
+#     #'LED8': '760', 'LED9': '880', 'LED10': '940', 'LED11': '1050', 'LED12': '1550','LED13': '340', 'LED14': '365'
+#     LED_TYPE: ['LED13']
+#     LED_TIME: [1.]
+#     # if open shutter effect, led time must be >=2.6s 
+#     shutter_effect: NO
+  
+
+  # flat_level: set the total skybackground value (e-) in the exptime,if none,set null, or delete the key
+  # flat_level_filt: the vale of "flat_level" is in the filter "flat_level_filt", can set NUV, u, g, r, i, z, y, if 
+  # none,set null,or delete the key
+
+  # sky_background:
+  #   exptime: 150. # [s]
+  #   shutter_effect: YES
+  #   flat_fielding: YES
+  #   enable_straylight_model: YES
+  #   flat_level: 20000
+  #   flat_level_filt: g
+
+  # Apply PRNU to accumulated photons
+  PRNU_effect: {}
+  # Accumulate photons caused by cosmic rays
+  cosmic_rays: {}
+  # Add Poission noise and dark current
+  poisson_and_dark:
+    add_dark: YES
+  # Simulate brighter fatter effects
+  bright_fatter: {}
+  # Add detector defects: hot/warm pixels, bad columns
+  detector_defects:
+    hot_pixels: YES
+    dead_pixels: YES
+    bad_columns: YES
+  # Apply response nonlinearity
+  nonlinearity: {}
+  # Apply CCD Saturation & Blooming
+  blooming: {}
+  # # Run CTE simulation
+  CTE_effect: {}
+  # Add prescan and overscan
+  prescan_overscan: {}
+  # Add bias
+  bias:
+    bias_16channel: YES
+  # Add readout noise
+  readout_noise: {}
+  # Apply gain
+  gain:
+    gain_16channel: YES
+  # Output the final image
+  quantization_and_output:
+    format_output: NO
+...
\ No newline at end of file

config/obs_config_Calibration.yaml → config/obs_config_Calibration_FLAT.yaml

@@ -3,39 +3,42 @@
 #
 #  Configuration file for CSST simulation
 #  For single exposure type:
-#                SCI-WIDE
+#                FLAT
 #  CSST-Sim Group, 2024/01/08
 #
 ###############################################
 
 # Observation type
-obs_type: "DARK"
-
+obs_type: "FLAT"
+obs_type_code: "128"
+obs_id: "00000001"
 # Define list of chips
-run_chips: [8]
+run_chips: [17]
 
 # Define observation sequence
 call_sequence:
   # Accumulate fluxes from objects
   led_calib_model:
     #"LED": ['LED1','LED2','LED3','LED4','LED5','LED6','LED7','LED8','LED9','LED10','LED11','LED12','LED13','LED14'] or null
-    #'LED1': '275', 'LED2': '310', 'LED3': '430', 'LED4': '505', 'LED5': '545', 'LED6': '590', 'LED7': '670',
-    #'LED8': '760', 'LED9': '940', 'LED10': '940', 'LED11': '1050', 'LED12': '1050','LED13': '340', 'LED14': '365'
-    LED_TYPE: ['LED5']
+    #'LED1': '275', 'LED2': '310', 'LED3': '430', 'LED4': '505', 'LED5': '525', 'LED6': '590', 'LED7': '670',
+    #'LED8': '760', 'LED9': '880', 'LED10': '940', 'LED11': '1050', 'LED12': '1550','LED13': '340', 'LED14': '365'
+    LED_TYPE: ['LED13']
     LED_TIME: [1.]
+    # if open shutter effect, led time must be >=2.6s 
+    shutter_effect: NO
   
 
   # flat_level: set the total skybackground value (e-) in the exptime,if none,set null, or delete the key
   # flat_level_filt: the vale of "flat_level" is in the filter "flat_level_filt", can set NUV, u, g, r, i, z, y, if 
   # none,set null,or delete the key
 
-  sky_background:
-    exptime: 150. # [s]
-    shutter_effect: YES
-    flat_fielding: YES
-    enable_straylight_model: YES
-    flat_level: 20000
-    flat_level_filt: g
+  # sky_background:
+  #   exptime: 150. # [s]
+  #   shutter_effect: YES
+  #   flat_fielding: YES
+  #   enable_straylight_model: YES
+  #   flat_level: 20000
+  #   flat_level_filt: g
 
   # Apply PRNU to accumulated photons
   PRNU_effect: {}

config/obs_config_SCI_WIDE_phot.yaml

@@ -14,7 +14,8 @@ obs_type_code: "101"
 obs_id: "00000001" # this setting will only be used if pointing list file is not given
 
 # Define list of chips
-run_chips: [8]
+run_chips: [6,7,8,9,11,12,13,14,15,16,17,18,19,20,22,23,24,25]
+#run_chips: [1,2,3,4,5,10,21,26,27,28,29,30]
 
 # Define observation sequence
 call_sequence:
@@ -66,7 +67,8 @@ call_sequence:
   # Run CTE simulation
   CTE_effect: {}
   # Add prescan and overscan
-  prescan_overscan: {}
+  prescan_overscan:
+    add_dark: YES
   # Add bias
   bias:
     bias_16channel: YES
@@ -78,4 +80,4 @@ call_sequence:
   # Output the final image
   quantization_and_output:
     format_output: YES
-...
\ No newline at end of file
+...

tests/test_darknoise_func.py

+import unittest
+
+import sys,os,math
+from itertools import islice
+import numpy as np
+import galsim
+import yaml
+
+from ObservationSim.Instrument import Chip, Filter, FilterParam, FocalPlane
+#from ObservationSim.Instrument.Chip import ChipUtils as chip_utils
+
+### test FUNCTION --- START ###
+def get_base_img(img, chip, read_noise, readout_time, dark_noise, exptime=150., InputDark=None):
+    if InputDark == None:
+        # base_level = read_noise**2 + dark_noise*(exptime+0.5*readout_time)
+        ## base_level = dark_noise*(exptime+0.5*readout_time)
+        base_level = dark_noise*(exptime)
+        base_img1 = base_level * np.ones_like(img.array)
+    else:
+        base_img1 = np.zeros_like(img.array)
+
+    ny = int(chip.npix_y/2)
+    nx = chip.npix_x
+    arr = np.arange(ny).reshape(ny, 1)
+    arr = np.broadcast_to(arr, (ny, nx))
+    base_img2 = np.zeros_like(img.array)
+    base_img2[:ny, :] = arr
+    base_img2[ny:, :] = arr[::-1,:]
+    base_img2[:,:]    = base_img2[:,:]*(readout_time/ny)*dark_noise
+    return base_img1+base_img2
+### test FUNCTION --- END ###
+
+
+def defineCCD(iccd, config_file):
+    with open(config_file, "r") as stream:
+        try:
+            config = yaml.safe_load(stream)
+            #for key, value in config.items():
+            #    print (key + " : " + str(value))
+        except yaml.YAMLError as exc:
+            print(exc)
+    chip = Chip(chipID=iccd, config=config)
+    chip.img = galsim.ImageF(chip.npix_x, chip.npix_y)
+    focal_plane = FocalPlane(chip_list=[iccd])
+    chip.img.wcs= focal_plane.getTanWCS(192.8595, 27.1283, -113.4333*galsim.degrees, chip.pix_scale)
+    return chip
+
+def defineFilt(chip):
+    filter_param = FilterParam()
+    filter_id, filter_type = chip.getChipFilter()
+    filt = Filter(
+        filter_id=filter_id,
+        filter_type=filter_type,
+        filter_param=filter_param,
+        ccd_bandpass=chip.effCurve)
+    bandpass_list = filt.bandpass_sub_list
+    return bandpass_list
+
+
+class detModule_coverage(unittest.TestCase):
+    def __init__(self, methodName='runTest'):
+        super(detModule_coverage, self).__init__(methodName)
+        self.dataPath = "/public/home/chengliang/CSSOSDataProductsSims/csst-simulation/tests/UNIT_TEST_DATA" ##os.path.join(os.getenv('UNIT_TEST_DATA_ROOT'), 'csst_fz_gc1')
+        self.iccd = 1
+
+    def test_add_dark(self):
+        config_file = os.path.join(self.dataPath, 'config_test.yaml')
+        chip = defineCCD(self.iccd, config_file)
+        bandpass = defineFilt(chip)
+        print(chip.chipID)
+        print(chip.cen_pix_x, chip.cen_pix_y)
+
+        exptime=150.
+        base_img = get_base_img(img=chip.img, chip=chip, read_noise=chip.read_noise, readout_time=chip.readout_time, dark_noise=chip.dark_noise, exptime=exptime, InputDark=None)
+
+        ny = int(chip.npix_y/2)
+        self.assertTrue( np.abs(np.max(base_img) - (exptime*chip.dark_noise+(ny-1)*(chip.readout_time/ny)*chip.dark_noise )) < 1e-6 )
+        self.assertTrue( np.min(base_img) == 3 )
+
+        base_img = get_base_img(img=chip.img, chip=chip, read_noise=chip.read_noise, readout_time=chip.readout_time, dark_noise=chip.dark_noise, exptime=150., InputDark="testTag")
+        self.assertTrue( np.abs(np.max(base_img) - ((ny-1)*(chip.readout_time/ny)*chip.dark_noise )) < 1e-6 )
+        
+
+
+if __name__ == '__main__':
+    unittest.main()