1.Modify fits header by defination;2.fix fgs bug: whether shutter effect open...

1.Modify fits header by defination;2.fix fgs bug: whether shutter effect open or close, fgs run error; 3. read pointinglist bug: jump the data of first row

ObservationSim/Config/ChipOutput.py

@@ -60,7 +60,7 @@ class ChipOutput(object):
         self.hdr += additional_column_names
     
     def create_output_file(self):
-        if self.pointing_type == 'SCIE':
+        if self.pointing_type == 'SCI':
             self.cat = open(os.path.join(self.subdir, self.cat_name), "w")
             self.logger.info("Creating catalog file %s ...\n"%(os.path.join(self.subdir, self.cat_name)))
             if not self.hdr.endswith("\n"):

ObservationSim/Config/Header/ImageHeader.py

@@ -22,6 +22,7 @@ from ObservationSim.Config._util import get_obs_id, get_file_type
 from datetime import datetime
 # import socket
 import platform
+import toml
 
 def chara2digit(char):
     """ Function to judge and convert characters to digitals
@@ -342,7 +343,7 @@ def WCS_def(xlen = 9216, ylen = 9232, gapy = 898.0, gapx1 = 534, gapx2 = 1309, r
 
 
 #TODO project_cycle is temporary， is not in header defined, delete in future
-def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec = -40, pixel_scale = 0.074, date='200930', time_obs='120000', im_type = 'SCIE', exptime=150., sat_pos = [0.,0.,0.], sat_vel = [0., 0., 0.], project_cycle=6, run_counter=0, chip_name="01"):
+def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec = -40, pixel_scale = 0.074, time_pt = None, im_type = 'SCI', exptime=150., sat_pos = [0.,0.,0.], sat_vel = [0., 0., 0.], project_cycle=6, run_counter=0, chip_name="01"):
 
     # array_size1, array_size2, flux, sigma = int(argv[1]), int(argv[2]), 1000.0, 5.0
 
@@ -350,7 +351,12 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec
     # k = (row_num-1)*6+col_num
     # ccdnum = str(k)
 
-    g_header_fn = os.path.split(os.path.realpath(__file__))[0] + '/global_header.header'
+    datetime_obs = datetime.utcfromtimestamp(time_pt)
+    datetime_obs = datetime_obs.replace(microsecond=round(datetime_obs.microsecond, -5))
+    # date_obs = datetime_obs.strftime("%y%m%d")
+    # time_obs = datetime_obs.strftime("%H%M%S%f")[:-5]
+
+    g_header_fn = os.path.split(os.path.realpath(__file__))[0] + '/csst_msc_l0_ms.fits'
     f = open(os.path.split(os.path.realpath(__file__))[0] + '/filter.lst')
     s = f.readline()
     s = s.strip("\n")
@@ -363,19 +369,33 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec
     s = s.strip("\n")
     CCDID = s.split()
 
-    h_prim = fits.Header()
-    h_prim = fits.Header.fromfile(g_header_fn)
+    # h_prim = fits.Header()
+    # h_prim = fits.Header.fromfile(g_header_fn)
+
+    header_fits = fits.open(g_header_fn)
+    h_prim = header_fits[0].header
+
+    # h_prim = fits.Header()
+    # with open(g_header_fn, 'r') as file:
+    #     header_toml = toml.load(file)
+    # h_key='HDU0'
+    # for key, value in header_toml[h_key].items():
+    #     h_card = fits.card.Card(header_toml[h_key][key]['key'],header_toml[h_key][key]['example'],header_toml[h_key][key]['comment'])
+    #     h_prim.append(h_card)
+
 
     # h_prim['PIXSIZE1'] = xlen
     # h_prim['PIXSIZE2'] = ylen
 
-    h_prim['DATE'] = '20'+date[0:2]+'-' + date[2:4]+'-'+date[4:6] + 'T' + time_obs[0:2]+':'+time_obs[2:4]+':'+time_obs[4:6]
+    h_prim['DATE'] = datetime_obs.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]
+    h_prim['DATE-OBS'] = datetime_obs.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]
+    # h_prim['DATE'] = '20'+date[0:2]+'-' + date[2:4]+'-'+date[4:6] + 'T' + time_obs[0:2]+':'+time_obs[2:4]+':'+time_obs[4:6]
     # h_prim['TIME'] = time_obs[0:2]+':'+time_obs[2:4]+':'+time_obs[4:6]
-    h_prim['DATE-OBS'] = '20'+date[0:2]+'-' + date[2:4]+'-'+date[4:6] + 'T' + time_obs[0:2]+':'+time_obs[2:4]+':'+time_obs[4:6]
+    # h_prim['DATE-OBS'] = '20'+date[0:2]+'-' + date[2:4]+'-'+date[4:6] + 'T' + time_obs[0:2]+':'+time_obs[2:4]+':'+time_obs[4:6]
     # h_prim['TIME-OBS'] = time_obs[0:2]+':'+time_obs[2:4]+':'+time_obs[4:6]
     # h_prim['DETECTOR'] = 'CHIP'+CCDID[k-1].rjust(2,'0')
-    h_prim['OBJ_RA'] = ra
-    h_prim['OBJ_DEC'] = dec
+    h_prim['RA_OBJ'] = ra
+    h_prim['DEC_OBJ'] = dec
 
     # obs_type = {'SCI': '01', 'BIAS': '03', 'DARK': '07', 'FLAT': '11', 'CRS': '98', 'CRD': '99'}
 
@@ -396,7 +416,7 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec
 
     co = coord.SkyCoord(ra, dec, unit='deg')
 
-    ra_hms = format(co.ra.hms.h, '02.0f') + format(co.ra.hms.m, '02.0f') + format(co.ra.hms.s, '02.0f')
+    ra_hms = format(co.ra.hms.h, '02.0f') + format(co.ra.hms.m, '02.0f') + format(co.ra.hms.s, '02.1f')
     dec_hms = format(co.dec.dms.d, '02.0f') + format(abs(co.dec.dms.m), '02.0f') + format(abs(co.dec.dms.s), '02.0f')
     h_prim['TARGET'] = ra_hms + '+' + dec_hms
     #
@@ -422,7 +442,8 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec
     h_prim['EXPEND'] = round(tend.mjd, 5)
     h_prim['CABEND'] = h_prim['EXPEND']
 
-    file_start_time = '20' + date[0:6] + time_obs[0:6]
+    # file_start_time = '20' + date[0:6] + time_obs[0:6]
+    file_start_time = datetime_obs.strftime("%Y%m%d%H%M%S")
     end_time_str = str(tend.datetime)
     file_end_time = end_time_str[0:4] + end_time_str[5:7]+end_time_str[8:10] + end_time_str[11:13] + end_time_str[14:16] + end_time_str[17:19]
     # h_prim['FILENAME'] = 'CSST_MSC_MS_' + im_type + '_' + file_start_time + '_' + file_end_time + '_' + OBS_id + '_' + CCDID[
@@ -445,7 +466,7 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec
     # h_prim['SIM_VER'] = (get_distribution("CSSTSim").version, "Version of CSST MSC simulation software")
     currentDateAndTime = datetime.now()
     compute_name = platform.node()
-    h_prim['FITSCREA'] = get_distribution("CSSTSim").version +'_' + currentDateAndTime.strftime("%Y%m%d") + '_' +compute_name
+    h_prim['FITSSWV'] = get_distribution("CSSTSim").version +'_' + currentDateAndTime.strftime("%Y%m%d") + '_' +compute_name
     h_prim['EPOCH'] = round((Time(h_prim['EXPSTART'], format='mjd', scale='tcb')).jyear, 1)
 
     return h_prim
@@ -453,7 +474,7 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec
 def generateExtensionHeader(chip, xlen = 9216, ylen = 9232,ra = 60, dec = -40, pa = -23.433, gain = 1.0, readout = 5.0, dark = 0.02, saturation=90000, pixel_scale = 0.074, pixel_size=1e-2, 
                             extName='SCIE', row_num = None, col_num = None, xcen=None, ycen=None, timestamp = 1621915200,exptime = 150., readoutTime = 40.):
 
-    e_header_fn = os.path.split(os.path.realpath(__file__))[0] + '/extension_header.header'
+    e_header_fn = os.path.split(os.path.realpath(__file__))[0] + '/csst_msc_l0_ms.fits'
     f = open(os.path.split(os.path.realpath(__file__))[0] + '/filter.lst')
     s = f.readline()
     s = s.strip("\n")
@@ -468,13 +489,24 @@ def generateExtensionHeader(chip, xlen = 9216, ylen = 9232,ra = 60, dec = -40, p
 
     # k = (row_num - 1) * 6 + col_num
 
-    h_ext = fits.Header.fromfile(e_header_fn)
+    # h_ext = fits.Header.fromfile(e_header_fn)
+
+    header_fits = fits.open(e_header_fn)
+    h_ext = header_fits[1].header
+
+    # h_ext = fits.Header()
+    # with open(e_header_fn, 'r') as file:
+    #     header_toml = toml.load(file)
+    # h_key='HDU1'
+    # for key, value in header_toml[h_key].items():
+    #     h_card = fits.card.Card(header_toml[h_key][key]['key'],header_toml[h_key][key]['example'],header_toml[h_key][key]['comment'])
+    #     h_ext.append(h_card)
 
     # h_ext['CCDCHIP'] = CCDID[k - 1].rjust(2, '0')
     # h_ext['CCDLABEL'] = filters[k-1] + '-' + filterID[k-1]
     # h_ext['FILTER'] = filters[k-1]
     h_ext['CHIPID'] = str(chip.chipID).rjust(2, '0')
-    h_ext['CHIPLAB'] = chip.chip_name
+    h_ext['CHIPLABL'] = chip.chip_name
     h_ext['FILTER'] = chip.filter_type
     h_ext['NAXIS1'] = xlen
     h_ext['NAXIS2'] = ylen
@@ -521,13 +553,15 @@ def generateExtensionHeader(chip, xlen = 9216, ylen = 9232,ra = 60, dec = -40, p
     tstart = Time(datetime_obs)
     tstart_read = Time(tstart.mjd + exptime / 86400., format="mjd")
     tend_read = Time(tstart.mjd + (exptime + readoutTime) / 86400., format="mjd")
-    t_s1 = str(tstart_read.datetime).split()
-    h_ext['READT0'] = t_s1[0]+'T'+t_s1[1]
-    t_s2 = str(tend_read.datetime).split()
-    h_ext['READT1'] = t_s2[0] + 'T' + t_s2[1]
-
-
-
+    # t_s1 = str(tstart_read.datetime).split()
+    # h_ext['ROTIME0'] = t_s1[0]+'T'+t_s1[1]
+    # t_s2 = str(tend_read.datetime).split()
+    # h_ext['ROTIME1'] = t_s2[0] + 'T' + t_s2[1]
+
+    tstart1=tstart.datetime.replace(microsecond=round(tstart.datetime.microsecond, -5))
+    h_ext['ROTIME0'] = tstart1.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]
+    tend_read1 = tend_read.datetime.replace(microsecond=round(tend_read.datetime.microsecond, -5))
+    h_ext['ROTIME1'] = tend_read1.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]
     # h_ext['POS_ANG'] = pa
     header_wcs = WCS_def(xlen=xlen, ylen=ylen, gapy=898.0, gapx1=534, gapx2=1309, ra_ref=ra, dec_ref=dec, pa=pa, pixel_scale=pixel_scale, pixel_size=pixel_size, 
                          rotate_chip=chip.rotate_angle, filter = h_ext['FILTER'], row_num=row_num, col_num=col_num, xcen = xcen, ycen = ycen)
@@ -545,6 +579,9 @@ def generateExtensionHeader(chip, xlen = 9216, ylen = 9232,ra = 60, dec = -40, p
     h_ext['CTYPE1'] = header_wcs['CTYPE1']
     h_ext['CTYPE2'] = header_wcs['CTYPE2']
 
+    h_ext['EXTNAME'] = 'IMAGE'
+    h_ext.comments["XTENSION"] = "image extension"
+
     return h_ext
 
 

ObservationSim/Config/Pointing.py

@@ -43,7 +43,7 @@ class Pointing(object):
                     return max(150., self.exp_time) # [TODO] for FGS
 
 
-    def read_pointing_columns(self, columns, id=0, t=1621915200, pointing_type='SCIE'):
+    def read_pointing_columns(self, columns, id=0, t=1621915200, pointing_type='SCI'):
         self.id = id
         col_len = len(columns)
         self.ra = float(columns[0])
@@ -72,7 +72,7 @@ class Pointing(object):
             
             # Load the configuration file for this particular pointing
             # [TODO]
-            self.obs_config_file = "/share/home/fangyuedong/20231211/csst-simulation/config/obs_config_SCI_WIDE_phot.yaml"
+            self.obs_config_file = "/home/zhangxin/CSST_SIM/CSST_sim_scheduler/csst-simulation/config/obs_config_SCI_WIDE_phot.yaml"
             with open(self.obs_config_file, "r") as stream:
                 try:
                     self.obs_param = yaml.safe_load(stream)

ObservationSim/Config/_util.py

 
-def get_obs_id(img_type='SCIE', project_cycle=6, run_counter=0, pointing_num=0):
+def get_obs_id(img_type='SCI', project_cycle=6, run_counter=0, pointing_num=0):
     # obs_type = {'SCI': '01', 'BIAS': '03', 'DARK': '07', 'FLAT': '11', 'CRS': '98', 'CRD': '99'}
-    obs_type = {'SCIE': '01', 'BIAS': '03', 'DARK': '07', 'FLAT': '11', 'CRS': '98', 'CRD': '99', 'CAL': '01'}
+    obs_type = {'SCI': '01', 'BIAS': '03', 'DARK': '07', 'FLAT': '11', 'CRS': '98', 'CRD': '99', 'CAL': '01'}
     obs_id = '1'+ obs_type[img_type] + str(int(project_cycle)).rjust(2, '0') + str(int(run_counter)) + str(pointing_num).rjust(8,'0')
     return obs_id
 

ObservationSim/ObservationSim.py

@@ -40,6 +40,9 @@ class Observation(object):
         
         # Get flat, shutter, and PRNU images
         chip.flat_img, _ = chip_utils.get_flat(img=chip.img, seed=int(self.config["random_seeds"]["seed_flat"]))
+        if chip.chipID > 30:
+            chip.shutter_img = np.ones_like(chip.img.array)
+        else:
             chip.shutter_img = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735, dt=1E-3)
         chip.prnu_img = Effects.PRNU_Img(xsize=chip.npix_x, ysize=chip.npix_y, sigma=0.01,
                                          seed=int(self.config["random_seeds"]["seed_prnu"]+chip.chipID))

ObservationSim/SimSteps.py

@@ -23,9 +23,7 @@ class SimSteps:
         self.all_filters = all_filters
     
     def prepare_headers(self, chip, pointing):
-        datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
-        date_obs = datetime_obs.strftime("%y%m%d")
-        time_obs = datetime_obs.strftime("%H%M%S")
+        
         self.h_prim = generatePrimaryHeader(
             xlen=chip.npix_x, 
             ylen=chip.npix_y, 
@@ -33,8 +31,7 @@ class SimSteps:
             ra=pointing.ra, 
             dec=pointing.dec, 
             pixel_scale=chip.pix_scale,
-            date=date_obs,
-            time_obs=time_obs,
+            time_pt = pointing.timestamp,
             exptime=pointing.exp_time,
             im_type=pointing.pointing_type,
             sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z],
@@ -432,6 +429,7 @@ class SimSteps:
         hdu2.header.comments['XTENSION'] = 'extension type'
         hdu2.header.comments['CHECKSUM'] = 'HDU checksum'
         hdu2.header.comments['DATASUM'] = 'data unit checksum'
+        hdu2.header.comments["XTENSION"] = "image extension"
         hdu1 = fits.HDUList([hdu1, hdu2])
         fname = os.path.join(self.chip_output.subdir, self.h_prim['FILENAME'] + '.fits')
         hdu1.writeto(fname, output_verify='ignore', overwrite=True)

ObservationSim/_util.py

@@ -36,10 +36,12 @@ def generate_pointing_list(config, pointing_filename=None, data_dir=None):
     if pointing_filename and data_dir:
         pointing_file = os.path.join(data_dir, pointing_filename)
         f = open(pointing_file, 'r')
-        for _ in range(1):
-            header = f.readline()
+        # for _ in range(1):
+        #     header = f.readline()
         iline = 0
         for line in f:
+            if len(line.strip()) == 0 or line[0]=='#':
+                continue
             if run_pointings and iline not in run_pointings:
                 iline += 1
                 ipoint += 1

config/obs_config_SCI_WIDE_phot.yaml

@@ -9,7 +9,7 @@
 ###############################################
 
 # Observation type
-obs_type: "SCIE"
+obs_type: "SCI"
 
 # Define list of chips
 run_chips: [6, 7, 8, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25]