diff --git a/ObservationSim/Config/ChipOutput.py b/ObservationSim/Config/ChipOutput.py index c4d2421b6e84321dc109f542c1d56735ac874da7..d3d8ff55e434aeee51c173959026c270fcb4e052 100755 --- a/ObservationSim/Config/ChipOutput.py +++ b/ObservationSim/Config/ChipOutput.py @@ -1,7 +1,7 @@ import os class ChipOutput(object): - def __init__(self, config, focal_plane, chip, filt, imgKey0="", imgKey1="", imgKey2="", exptime=150., mjdTime="", ra_cen=None, dec_cen=None, pointing_ID='0', subdir="./", prefix=""): + def __init__(self, config, focal_plane, chip, filt, imgKey0="", imgKey1="", imgKey2="", exptime=150., mjdTime="", ra_cen=None, dec_cen=None, pointing_type='MS', pointing_ID='0', subdir="./", prefix=""): self.focal_plane = focal_plane self.chip = chip self.filt = filt @@ -25,20 +25,21 @@ class ChipOutput(object): # hdr1 = "#ID ID_chip filter xImage yImage ra dec z mag flag SNR " hdr1 = "#ID ID_chip filter xImage yImage ra dec z mag flag " - hdr2 = "thetaR bfrac hlr_disk hlr_bulge e1_disk e2_disk e1_bulge e2_bulge e1_total e2_total" - hdr3 = "e1PSF e2PSF e1 e2 g1 g2 e1OBS e2OBS" + hdr2 = "thetaR bfrac hlr_disk hlr_bulge e1_disk e2_disk e1_bulge e2_bulge " + hdr3 = "e1PSF e2PSF e1 e2 g1 g2 e1OBS e2OBS " hdr4 = "sed_type av redden " hdr5 = "star_model teff logg feh\n" # fmt1 = "%10d %4d %5s %10.3f %10.3f %15.6f %15.6f %7.4f %8.4f %2d %9.2f " fmt1 = "%10d %4d %5s %10.3f %10.3f %15.6f %15.6f %7.4f %8.4f %2d " - fmt2 = "%8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f" - fmt3 = "%8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f" + fmt2 = "%8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f " + fmt3 = "%8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f " fmt4 = "%2d %8.4f %8.4f " fmt5 = "%10s %8.4f %8.4f %8.4f\n" self.hdr = hdr1 + hdr2 + hdr3 + hdr4 + hdr5 self.fmt = fmt1 + fmt2 + fmt3 + fmt4 + fmt5 - self.cat = open(os.path.join(self.subdir, self.cat_name), "w") - self.cat.write(self.hdr) + if pointing_type == 'MS': + self.cat = open(os.path.join(self.subdir, self.cat_name), "w") + self.cat.write(self.hdr) def updateHDR(self, hdr): hdrNew = [{"name":"RDNOISE", "value":self.chip.read_noise, "comment":"read noise in e-/pixel"}, @@ -79,15 +80,14 @@ class ChipOutput(object): e1, e2, g1, g2, e1OBS, e2OBS = obj.getObservedEll(g1, g2) if obj.type == 'galaxy': line = self.fmt%(obj.id, int(self.chipLabel), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.z, obj.getMagFilter(self.filt), obj.param["star"], obj.thetaR, obj.bfrac, obj.hlr_disk, obj.hlr_bulge, - obj.e1_disk, obj.e2_disk, obj.e1_bulge, obj.e2_bulge, obj.e1_total, obj.e2_total, + obj.e1_disk, obj.e2_disk, obj.e1_bulge, obj.e2_bulge, pos_shear.g1, pos_shear.g2, e1, e2, g1, g2, e1OBS, e2OBS, obj.sed_type, obj.param['av'], obj.param['redden'], 'n', 0, 0, 0) elif obj.type == "quasar": line = self.fmt % (obj.id, int(self.chipLabel), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.z, obj.getMagFilter(self.filt), obj.param["star"], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, pos_shear.g1, pos_shear.g2, e1, e2, g1, g2, e1OBS, e2OBS, obj.sed_type, obj.param['av'], obj.param['redden'], 'n', 0.0, 0.0, 0.0) else: - line = self.fmt%(obj.id, int(self.chipLabel), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.z, obj.getMagFilter(self.filt), obj.param["star"], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + line = self.fmt%(obj.id, int(self.chipLabel), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.z, obj.getMagFilter(self.filt), obj.param["star"], 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, pos_shear.g1, pos_shear.g2, e1, e2, g1, g2, e1OBS, e2OBS, 0, 0.0, 0.0, obj.param['model_tag'], obj.param['teff'], obj.param['logg'],obj.param['feh']) # line = self.fmt%(obj.id, int(self.chipLabel), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.z, obj.getMagFilter(self.filt), obj.param["star"], pos_shear.g1, pos_shear.g2, e1, e2, g1, g2, e1OBS, e2OBS) self.cat.write(line) diff --git a/ObservationSim/Config/Config.py b/ObservationSim/Config/Config.py index 02a62798898b69df4e9d11fcf063b772b92dff19..140f143b2432256c61ca4dafa218bac787749895 100755 --- a/ObservationSim/Config/Config.py +++ b/ObservationSim/Config/Config.py @@ -24,9 +24,9 @@ def ConfigDir(cat_dir=None, work_dir=None, data_dir=None, config_file_path=None) # path_dict["output_cat_dir"] = os.path.join(path_dict["work_dir"], "scat/") # if not os.path.exists(path_dict["output_cat_dir"]): # os.system("mkdir %s"%path_dict["output_cat_dir"]) - path_dict["output_img_dir"] = os.path.join(path_dict["work_dir"], "simg/") - if not os.path.exists(path_dict["output_img_dir"]): - os.system("mkdir %s"%path_dict["output_img_dir"]) + # path_dict["output_img_dir"] = os.path.join(path_dict["work_dir"], "simg/") + # if not os.path.exists(path_dict["output_img_dir"]): + # os.system("mkdir %s"%path_dict["output_img_dir"]) # Data directory if data_dir == None: diff --git a/ObservationSim/Config/Header/ImageHeader.py b/ObservationSim/Config/Header/ImageHeader.py index 8a60d29d24b1cbaff5c03a24d239dcd71cfea69a..420caf0966cd464547f5dee9487d304b37aa52b9 100644 --- a/ObservationSim/Config/Header/ImageHeader.py +++ b/ObservationSim/Config/Header/ImageHeader.py @@ -233,7 +233,7 @@ def WCS_def(xlen = 9216, ylen = 9232, gapy = 898.0, gapx1 = 534, gapx2 = 1309, r -def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec = -40, psize = 0.074, row_num = 1, col_num = 1, date='200930', time_obs='120000', im_type = 'MS'): +def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec = -40, psize = 0.074, row_num = 1, col_num = 1, date='200930', time_obs='120000', im_type = 'MS', exptime=150.): # array_size1, array_size2, flux, sigma = int(argv[1]), int(argv[2]), 1000.0, 5.0 @@ -262,12 +262,19 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec h_prim['FILENAME'] = 'MSC_' + im_type + '_' + date + time_obs + '_1' + pointNum.rjust(8,'0') + '_' +CCDID[k-1].rjust(2,'0')+'_raw' h_prim['DATE'] = '20'+date[0:2]+'-' + date[2:4]+'-'+date[4:6] h_prim['TIME'] = time_obs[0:2]+':'+time_obs[2:4]+':'+time_obs[4:6] + h_prim['DATE-OBS'] = '20'+date[0:2]+'-' + date[2:4]+'-'+date[4:6] + h_prim['TIME-OBS'] = time_obs[0:2]+':'+time_obs[2:4]+':'+time_obs[4:6] h_prim['DETECTOR'] = 'CCD'+CCDID[k-1].rjust(2,'0') h_prim['RA_OBJ'] = ra h_prim['DEC_OBJ'] = dec h_prim['OBJECT'] = '1'+ pointNum.rjust(8,'0') h_prim['OBSID'] = '1'+ pointNum.rjust(8,'0') h_prim['TELFOCUS'] = 'f/14' + h_prim['EXPTIME'] = exptime + + # Define file types + file_type = {'MS':'sci', 'CLB':'zero', 'CLD':'dark', 'CLF':'flat', 'CRS':'cosmic_ray', 'CRD':'cosmic_ray'} + h_prim['FILETYPE'] = file_type[im_type] co = coord.SkyCoord(ra, dec, unit='deg') @@ -277,13 +284,14 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointNum = '1', ra = 60, dec h_prim['RA_NOM'] = ra_hms h_prim['DEC_NOM'] = dec_hms - h_prim['PIXSCAL1'] = str(psize) - h_prim['PIXSCAL2'] = str(psize) + h_prim['PIXSCAL1'] = psize + h_prim['PIXSCAL2'] = psize ttt = h_prim['DATE'] + 'T' + h_prim['TIME'] tstart = Time(ttt) h_prim['EXPSTART'] = round(tstart.mjd, 5) - tend = Time(tstart.cxcsec + h_prim['EXPTIME'], format="cxcsec") + # tend = Time(tstart.cxcsec + h_prim['EXPTIME'], format="cxcsec") + tend = Time(tstart.mjd + h_prim['EXPTIME']/86400., format="mjd") h_prim['EXPEND'] = round(tend.mjd, 5) return h_prim @@ -315,6 +323,7 @@ def generateExtensionHeader(xlen = 9216, ylen = 9232,ra = 60, dec = -40, pa = -2 h_ext['GAIN1'] = gain h_ext['RDNOISE1'] = readout h_ext['CCDCHIP'] = 'ccd' + CCDID[k-1].rjust(2,'0') + h_ext['POS_ANG'] = pa header_wcs = WCS_def(xlen=xlen, ylen=ylen, gapy=898.0, gapx1=534, gapx2=1309, ra=ra, dec=dec, pa=pa, psize=psize, row_num=row_num, col_num=col_num) diff --git a/ObservationSim/Config/Header/__pycache__/ImageHeader.cpython-38.pyc b/ObservationSim/Config/Header/__pycache__/ImageHeader.cpython-38.pyc index 4d6d0d9e0d1031c900efa02ff7aaca5a99966411..5c84dc417479dd9ae4bc38e39da70e0cabcf3e93 100644 Binary files a/ObservationSim/Config/Header/__pycache__/ImageHeader.cpython-38.pyc and b/ObservationSim/Config/Header/__pycache__/ImageHeader.cpython-38.pyc differ diff --git a/ObservationSim/Config/Header/__pycache__/__init__.cpython-38.pyc b/ObservationSim/Config/Header/__pycache__/__init__.cpython-38.pyc index 622807c66edb4c6a24df01fc89e040ebdb925b8d..3307ee5e9eb502bd404ae7bbff5bc360c9cde375 100644 Binary files a/ObservationSim/Config/Header/__pycache__/__init__.cpython-38.pyc and b/ObservationSim/Config/Header/__pycache__/__init__.cpython-38.pyc differ diff --git a/ObservationSim/Config/__pycache__/ChipOutput.cpython-38.pyc b/ObservationSim/Config/__pycache__/ChipOutput.cpython-38.pyc index e2d55f303aceaec8fcfffcb91c49315d5cbff2fc..8629f5acd8af472c3d00ab6b178614b1411845fb 100644 Binary files a/ObservationSim/Config/__pycache__/ChipOutput.cpython-38.pyc and b/ObservationSim/Config/__pycache__/ChipOutput.cpython-38.pyc differ diff --git a/ObservationSim/Config/__pycache__/Config.cpython-38.pyc b/ObservationSim/Config/__pycache__/Config.cpython-38.pyc index c53335da667c50ebefe8d059c64a049c4f41d1d8..90677a327f1f5aa30f55233c1c180be2926dd1bf 100644 Binary files a/ObservationSim/Config/__pycache__/Config.cpython-38.pyc and b/ObservationSim/Config/__pycache__/Config.cpython-38.pyc differ diff --git a/ObservationSim/Config/__pycache__/__init__.cpython-38.pyc b/ObservationSim/Config/__pycache__/__init__.cpython-38.pyc index e72fec1603550e6945c7eeec1b444125a397a609..383f859df993ea3782b8c1fd2ac7101360974e00 100644 Binary files a/ObservationSim/Config/__pycache__/__init__.cpython-38.pyc and b/ObservationSim/Config/__pycache__/__init__.cpython-38.pyc differ diff --git a/ObservationSim/Instrument/Chip/Chip.py b/ObservationSim/Instrument/Chip/Chip.py index 9df960b5bc71268074998010008100d4be417370..f3d518f310956b92bc2ca239c60ba1f2dbadc591 100755 --- a/ObservationSim/Instrument/Chip/Chip.py +++ b/ObservationSim/Instrument/Chip/Chip.py @@ -5,6 +5,9 @@ import Instrument.Chip.Effects as effects from Instrument.FocalPlane import FocalPlane from astropy.table import Table from numpy.random import Generator, PCG64 +from Config.Header import generatePrimaryHeader, generateExtensionHeader +from astropy.io import fits +from datetime import datetime class Chip(FocalPlane): def __init__(self, chipID, ccdEffCurve_dir, CRdata_dir, normalize_dir=None, sls_dir=None, config=None, treering_func=None): @@ -33,6 +36,10 @@ class Chip(FocalPlane): self.bias_level = float(config["bias_level"]) self.overscan = 1000 self.exptime = 150 # second + # self.dark_exptime = 300 # second + # self.flat_exptime = 150 # second + self.dark_exptime = float(config['dark_exptime']) + self.flat_exptime = float(config['flat_exptime']) # A chip ID must be assigned self.chipID = int(chipID) @@ -297,7 +304,54 @@ class Chip(FocalPlane): if self.chipID == 30: confFile = ['CSST_GI8.conf', 'CSST_GI7.conf'] return confFile - def addEffects(self, config, img, chip_output, filt, exptime=150., pointing_ID=0): + def generateHeader(self, ra_cen, dec_cen, img_rot, im_type, pointing_ID, date_obs, time_obs, exptime=150.): + h_prim = generatePrimaryHeader( + xlen=self.npix_x, + ylen=self.npix_y, + pointNum = str(pointing_ID), + ra=ra_cen, + dec=dec_cen, + psize=self.pix_scale, + row_num=self.rowID, + col_num=self.colID, + date=date_obs, + time_obs=time_obs, + im_type = im_type, + exptime=exptime + ) + h_ext = generateExtensionHeader( + xlen=self.npix_x, + ylen=self.npix_y, + ra=ra_cen, + dec=dec_cen, + pa=img_rot.deg, + gain=self.gain, + readout=self.read_noise, + dark=self.dark_noise, + saturation=90000, + psize=self.pix_scale, + row_num=self.rowID, + col_num=self.colID, + extName='raw') + return h_prim, h_ext + + def outputCal(self, img, ra_cen, dec_cen, img_rot, im_type, pointing_ID, date_obs, time_obs, output_dir, exptime=150.): + h_prim, h_ext = self.generateHeader( + ra_cen=ra_cen, + dec_cen=dec_cen, + img_rot=img_rot, + im_type=im_type, + pointing_ID=pointing_ID, + date_obs=date_obs, + time_obs=time_obs, + exptime=exptime) + hdu1 = fits.PrimaryHDU(header=h_prim) + hdu2 = fits.ImageHDU(img.array, header=h_ext) + hdu1 = fits.HDUList([hdu1, hdu2]) + fname = os.path.join(output_dir, h_prim['FILENAME']+'.fits') + hdu1.writeto(fname, output_verify='ignore', overwrite=True) + + def addEffects(self, config, img, chip_output, filt, ra_cen, dec_cen, img_rot, exptime=150., pointing_ID=0, timestamp_obs=1621915200, pointing_type='MS'): SeedGainNonuni=int(config["seed_gainNonUniform"]) SeedBiasNonuni=int(config["seed_biasNonUniform"]) SeedRnNonuni = int(config["seed_rnNonUniform"]) @@ -356,7 +410,7 @@ class Chip(FocalPlane): img += self.dark_noise*exptime # Add cosmic-rays - if config["cosmic_ray"].lower() == "y": + if config["cosmic_ray"].lower() == "y" and pointing_type=='MS': print(" Adding Cosmic-Ray", flush=True) cr_map = effects.produceCR_Map( xLen=self.npix_x, yLen=self.npix_y, @@ -364,13 +418,28 @@ class Chip(FocalPlane): cr_pixelRatio=0.003, gain=self.gain, attachedSizes=self.attachedSizes, - seed=SeedCosmicRay+pointing_ID*30+self.chipID) + seed=SeedCosmicRay+pointing_ID*30+self.chipID) # seed: obj-imaging:+0; bias:+1; dark:+2; flat:+3; img += cr_map cr_map[cr_map > 65535] = 65535 cr_map[cr_map < 0] = 0 crmap_gsimg = galsim.Image(cr_map, dtype=np.uint16) + del cr_map # crmap_gsimg.write("%s/CosmicRay_%s_1.fits" % (chip_output.subdir, self.chipID)) - crmap_gsimg.write("%s/CosmicRay_%s.fits" % (chip_output.subdir, self.chipID)) + # crmap_gsimg.write("%s/CosmicRay_%s.fits" % (chip_output.subdir, self.chipID)) + datetime_obs = datetime.fromtimestamp(timestamp_obs) + date_obs = datetime_obs.strftime("%y%m%d") + time_obs = datetime_obs.strftime("%H%M%S") + self.outputCal( + img=crmap_gsimg, + ra_cen=ra_cen, + dec_cen=dec_cen, + img_rot=img_rot, + im_type='CRS', + pointing_ID=pointing_ID, + date_obs=date_obs, + time_obs=time_obs, + output_dir=chip_output.subdir, + exptime=150.) del crmap_gsimg # Add Bias level @@ -383,7 +452,7 @@ class Chip(FocalPlane): seed=SeedBiasNonuni+self.chipID) # Bias output - if config["bias_output"].lower() == "y": + if config["bias_output"].lower() == "y" and pointing_type=='CAL': print(" Output N frame Bias files", flush=True) NBias = int(config["NBias"]) for i in range(NBias): @@ -392,7 +461,19 @@ class Chip(FocalPlane): bias_level=float(config["bias_level"]), ncombine=1, read_noise=self.read_noise, gain=1, seed=SeedBiasNonuni+self.chipID) - + if config["cosmic_ray"].lower() == "y": + if config["cray_differ"].lower() == "y": + cr_map = effects.produceCR_Map( + xLen=self.npix_x, yLen=self.npix_y, + exTime=0.01, + cr_pixelRatio=0.003*0.01/self.exptime, + gain=self.gain, + attachedSizes=self.attachedSizes, + seed=SeedCosmicRay+pointing_ID*30+self.chipID+1) + # seed: obj-imaging:+0; bias:+1; dark:+2; flat:+3; + BiasCombImg += cr_map + del cr_map + # Non-Linearity for Bias if config["non_linear"].lower() == "y": print(" Applying Non-Linearity on the Bias image", flush=True) @@ -408,11 +489,26 @@ class Chip(FocalPlane): BiasCombImg.replaceNegative(replace_value=0) BiasCombImg.quantize() BiasCombImg = galsim.ImageUS(BiasCombImg) - BiasCombImg.write("%s/BiasImg_%s_%s_%s.fits" % (chip_output.subdir, BiasTag, self.chipID, i+1)) + # BiasCombImg.write("%s/BiasImg_%s_%s_%s.fits" % (chip_output.subdir, BiasTag, self.chipID, i+1)) + datetime_obs = datetime.fromtimestamp(timestamp_obs) + date_obs = datetime_obs.strftime("%y%m%d") + time_obs = datetime_obs.strftime("%H%M%S") + timestamp_obs += 5 * 60 + self.outputCal( + img=BiasCombImg, + ra_cen=ra_cen, + dec_cen=dec_cen, + img_rot=img_rot, + im_type='CLB', + pointing_ID=pointing_ID, + date_obs=date_obs, + time_obs=time_obs, + output_dir=chip_output.subdir, + exptime=0.0) del BiasCombImg # Export combined (ncombine, Vignetting + PRNU) & single vignetting flat-field file - if config["flat_output"].lower() == "y": + if config["flat_output"].lower() == "y" and pointing_type=='CAL': print(" Output N frame Flat-Field files", flush=True) NFlat = int(config["NFlat"]) if config["add_bias"].lower() == "y": @@ -421,7 +517,7 @@ class Chip(FocalPlane): elif config["add_bias"].lower() == "n": biaslevel = 0 overscan = 0 - darklevel = self.dark_noise*self.exptime + darklevel = self.dark_noise*self.flat_exptime for i in range(NFlat): FlatSingle = flat_img * prnu_img + darklevel FlatCombImg, FlatTag = effects.MakeFlatNcomb( @@ -434,7 +530,17 @@ class Chip(FocalPlane): seed_bias=SeedDefective+self.chipID ) if config["cosmic_ray"].lower() == "y": + if config["cray_differ"].lower() == "y": + cr_map = effects.produceCR_Map( + xLen=self.npix_x, yLen=self.npix_y, + exTime=self.flat_exptime, + cr_pixelRatio=0.003*self.flat_exptime/self.exptime, + gain=self.gain, + attachedSizes=self.attachedSizes, + seed=SeedCosmicRay+pointing_ID*30+self.chipID+3) + # seed: obj-imaging:+0; bias:+1; dark:+2; flat:+3; FlatCombImg += cr_map + del cr_map if config["non_linear"].lower() == "y": print(" Applying Non-Linearity on the Flat image", flush=True) @@ -459,7 +565,23 @@ class Chip(FocalPlane): FlatCombImg.replaceNegative(replace_value=0) FlatCombImg.quantize() FlatCombImg = galsim.ImageUS(FlatCombImg) - FlatCombImg.write("%s/FlatImg_%s_%s_%s.fits" % (chip_output.subdir, FlatTag, self.chipID, i+1)) + # FlatCombImg.write("%s/FlatImg_%s_%s_%s.fits" % (chip_output.subdir, FlatTag, self.chipID, i+1)) + datetime_obs = datetime.fromtimestamp(timestamp_obs) + date_obs = datetime_obs.strftime("%y%m%d") + time_obs = datetime_obs.strftime("%H%M%S") + timestamp_obs += 5 * 60 + self.outputCal( + img=FlatCombImg, + ra_cen=ra_cen, + dec_cen=dec_cen, + img_rot=img_rot, + im_type='CLF', + pointing_ID=pointing_ID, + date_obs=date_obs, + time_obs=time_obs, + output_dir=chip_output.subdir, + exptime=self.flat_exptime) + del FlatCombImg, FlatSingle, prnu_img # flat_img.replaceNegative(replace_value=0) # flat_img.quantize() @@ -467,7 +589,7 @@ class Chip(FocalPlane): del flat_img # Export Dark current images - if config["dark_output"].lower() == "y": + if config["dark_output"].lower() == "y" and pointing_type=='CAL': print(" Output N frame Dark Current files", flush=True) NDark = int(config["NDark"]) if config["add_bias"].lower() == "y": @@ -479,11 +601,39 @@ class Chip(FocalPlane): for i in range(NDark): DarkCombImg, DarkTag = effects.MakeDarkNcomb( self.npix_x, self.npix_y, - overscan=overscan, bias_level=biaslevel, darkpsec=0.02, exptime=150, + overscan=overscan, bias_level=biaslevel, darkpsec=0.02, exptime=self.dark_exptime, ncombine=1, read_noise=self.read_noise, gain=1, seed_bias=SeedBiasNonuni+self.chipID) if config["cosmic_ray"].lower() == "y": + if config["cray_differ"].lower() == "y": + cr_map = effects.produceCR_Map( + xLen=self.npix_x, yLen=self.npix_y, + exTime=self.dark_exptime, + cr_pixelRatio=0.003*self.dark_exptime/self.exptime, + gain=self.gain, + attachedSizes=self.attachedSizes, + seed=SeedCosmicRay+pointing_ID*30+self.chipID+2) + # seed: obj-imaging:+0; bias:+1; dark:+2; flat:+3; DarkCombImg += cr_map + cr_map[cr_map > 65535] = 65535 + cr_map[cr_map < 0] = 0 + crmap_gsimg = galsim.Image(cr_map, dtype=np.uint16) + del cr_map + datetime_obs = datetime.fromtimestamp(timestamp_obs) + date_obs = datetime_obs.strftime("%y%m%d") + time_obs = datetime_obs.strftime("%H%M%S") + self.outputCal( + img=crmap_gsimg, + ra_cen=ra_cen, + dec_cen=dec_cen, + img_rot=img_rot, + im_type='CRD', + pointing_ID=pointing_ID, + date_obs=date_obs, + time_obs=time_obs, + output_dir=chip_output.subdir, + exptime=self.dark_exptime) + del crmap_gsimg # Non-Linearity for Dark if config["non_linear"].lower() == "y": @@ -513,13 +663,24 @@ class Chip(FocalPlane): DarkCombImg.replaceNegative(replace_value=0) DarkCombImg.quantize() DarkCombImg = galsim.ImageUS(DarkCombImg) - DarkCombImg.write("%s/DarkImg_%s_%s_%s.fits" % (chip_output.subdir, DarkTag, self.chipID, i+1)) + # DarkCombImg.write("%s/DarkImg_%s_%s_%s.fits" % (chip_output.subdir, DarkTag, self.chipID, i+1)) + datetime_obs = datetime.fromtimestamp(timestamp_obs) + date_obs = datetime_obs.strftime("%y%m%d") + time_obs = datetime_obs.strftime("%H%M%S") + timestamp_obs += 5 * 60 + self.outputCal( + img=DarkCombImg, + ra_cen=ra_cen, + dec_cen=dec_cen, + img_rot=img_rot, + im_type='CLD', + pointing_ID=pointing_ID, + date_obs=date_obs, + time_obs=time_obs, + output_dir=chip_output.subdir, + exptime=self.dark_exptime) del DarkCombImg - # garbage collection of cosmic-ray array - if config["cosmic_ray"].lower() == "y": - del cr_map - # Apply Nonlinearity on the chip image if config["non_linear"].lower() == "y": print(" Applying Non-Linearity on the chip image", flush=True) diff --git a/ObservationSim/Instrument/Chip/__pycache__/Chip.cpython-38.pyc b/ObservationSim/Instrument/Chip/__pycache__/Chip.cpython-38.pyc index 5ee7f28c7b1505fa140bf6bb0134e74c39a46c24..714df698948c5c51688fad876725832c1f66b762 100644 Binary files a/ObservationSim/Instrument/Chip/__pycache__/Chip.cpython-38.pyc and b/ObservationSim/Instrument/Chip/__pycache__/Chip.cpython-38.pyc differ diff --git a/ObservationSim/Instrument/Chip/__pycache__/Effects.cpython-38.pyc b/ObservationSim/Instrument/Chip/__pycache__/Effects.cpython-38.pyc index 3ae921f3f4af7e561cb845bcbe93df7de48819bd..44f89fdced2fa9a27886b382325bbac16bb0f113 100644 Binary files a/ObservationSim/Instrument/Chip/__pycache__/Effects.cpython-38.pyc and b/ObservationSim/Instrument/Chip/__pycache__/Effects.cpython-38.pyc differ diff --git a/ObservationSim/Instrument/Chip/__pycache__/__init__.cpython-38.pyc b/ObservationSim/Instrument/Chip/__pycache__/__init__.cpython-38.pyc index 9535fe81ce0854aeecce71312a8625177ed570b1..6202baf53621573af1f61b61635ba36e5ab439d4 100644 Binary files a/ObservationSim/Instrument/Chip/__pycache__/__init__.cpython-38.pyc and b/ObservationSim/Instrument/Chip/__pycache__/__init__.cpython-38.pyc differ diff --git a/ObservationSim/Instrument/__pycache__/Filter.cpython-38.pyc b/ObservationSim/Instrument/__pycache__/Filter.cpython-38.pyc index 7fc9c08be238e5564b97b663fdc58bf50a467233..25130bb497d27f287065bb9c8502433ea6743b78 100644 Binary files a/ObservationSim/Instrument/__pycache__/Filter.cpython-38.pyc and b/ObservationSim/Instrument/__pycache__/Filter.cpython-38.pyc differ diff --git a/ObservationSim/Instrument/__pycache__/FilterParam.cpython-38.pyc b/ObservationSim/Instrument/__pycache__/FilterParam.cpython-38.pyc index 1ef710512c2b2dcaf9c3fbf72969f4c74f24b7c9..1ca75d6fbf3f5c70a59905b25f1d3ae636dae5c9 100644 Binary files a/ObservationSim/Instrument/__pycache__/FilterParam.cpython-38.pyc and b/ObservationSim/Instrument/__pycache__/FilterParam.cpython-38.pyc differ diff --git a/ObservationSim/Instrument/__pycache__/FocalPlane.cpython-38.pyc b/ObservationSim/Instrument/__pycache__/FocalPlane.cpython-38.pyc index 7149f30e3622a1b3455a0b300e6d88e741bdfb30..92e6fc32170edaead10e083be1ccdcfe1fdec6dd 100644 Binary files a/ObservationSim/Instrument/__pycache__/FocalPlane.cpython-38.pyc and b/ObservationSim/Instrument/__pycache__/FocalPlane.cpython-38.pyc differ diff --git a/ObservationSim/Instrument/__pycache__/Telescope.cpython-38.pyc b/ObservationSim/Instrument/__pycache__/Telescope.cpython-38.pyc index 8162f83c83d34fc513055dfe6f09813b070579df..d3af64c8e58419a840b314c4892e91ccafa0c97d 100644 Binary files a/ObservationSim/Instrument/__pycache__/Telescope.cpython-38.pyc and b/ObservationSim/Instrument/__pycache__/Telescope.cpython-38.pyc differ diff --git a/ObservationSim/Instrument/__pycache__/__init__.cpython-38.pyc b/ObservationSim/Instrument/__pycache__/__init__.cpython-38.pyc index 183226fc3b7215c4d417e4bfed57633383f0c8cd..cf5ac354c6883e23b732c60621361dbe7f924d8b 100644 Binary files a/ObservationSim/Instrument/__pycache__/__init__.cpython-38.pyc and b/ObservationSim/Instrument/__pycache__/__init__.cpython-38.pyc differ diff --git a/ObservationSim/Instrument/__pycache__/_util.cpython-38.pyc b/ObservationSim/Instrument/__pycache__/_util.cpython-38.pyc index 2945fc154ecdbc208cde8d2a83c63eab2d58c570..87890f7b3a2c1a8b757004a39b4c354217d17c7c 100644 Binary files a/ObservationSim/Instrument/__pycache__/_util.cpython-38.pyc and b/ObservationSim/Instrument/__pycache__/_util.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/SpecDisperser/__pycache__/SpecDisperser.cpython-38.pyc b/ObservationSim/MockObject/SpecDisperser/__pycache__/SpecDisperser.cpython-38.pyc index 6b677b942ca8c14bc62be3bd74f994a5283820e5..6c273a84ba7338c4306e2e5bedb66b3c57a23655 100644 Binary files a/ObservationSim/MockObject/SpecDisperser/__pycache__/SpecDisperser.cpython-38.pyc and b/ObservationSim/MockObject/SpecDisperser/__pycache__/SpecDisperser.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/SpecDisperser/__pycache__/__init__.cpython-38.pyc b/ObservationSim/MockObject/SpecDisperser/__pycache__/__init__.cpython-38.pyc index 69cfc0e7fb8050291252554d9c59de4f3b3d5199..cd38720add740402e19da900411cfd48b385dfd6 100644 Binary files a/ObservationSim/MockObject/SpecDisperser/__pycache__/__init__.cpython-38.pyc and b/ObservationSim/MockObject/SpecDisperser/__pycache__/__init__.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/SpecDisperser/disperse_c/__pycache__/__init__.cpython-38.pyc b/ObservationSim/MockObject/SpecDisperser/disperse_c/__pycache__/__init__.cpython-38.pyc index bbbe05f79f8e7a8db2aa23c82e432b729d289fba..def3ba9e2a42b77f17b8285b878c5815b52b8b18 100644 Binary files a/ObservationSim/MockObject/SpecDisperser/disperse_c/__pycache__/__init__.cpython-38.pyc and b/ObservationSim/MockObject/SpecDisperser/disperse_c/__pycache__/__init__.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/__pycache__/Catalog.cpython-38.pyc b/ObservationSim/MockObject/__pycache__/Catalog.cpython-38.pyc index 7d7ed96c7e60d875cf77bb3bc5bc501ce47c5c6d..10f5c371fa5a6f4fff0530887acb42acd33c3b93 100644 Binary files a/ObservationSim/MockObject/__pycache__/Catalog.cpython-38.pyc and b/ObservationSim/MockObject/__pycache__/Catalog.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/__pycache__/Galaxy.cpython-38.pyc b/ObservationSim/MockObject/__pycache__/Galaxy.cpython-38.pyc index d55f457c5f18f9ed2ae74b42403c07d37560f483..6f7f272fcf2d884910242f1d4d33794aa8437ec7 100644 Binary files a/ObservationSim/MockObject/__pycache__/Galaxy.cpython-38.pyc and b/ObservationSim/MockObject/__pycache__/Galaxy.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/__pycache__/MockObject.cpython-38.pyc b/ObservationSim/MockObject/__pycache__/MockObject.cpython-38.pyc index 0940990a16377bf8178d2c8b9c1e7b4654af8910..581be6e18f28f80061bd082dd220d1837ac7cabd 100644 Binary files a/ObservationSim/MockObject/__pycache__/MockObject.cpython-38.pyc and b/ObservationSim/MockObject/__pycache__/MockObject.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/__pycache__/Quasar.cpython-38.pyc b/ObservationSim/MockObject/__pycache__/Quasar.cpython-38.pyc index 7ed17f6af14407079eeb46a5a91f87c9abcf7609..70215de2647aae7339d7bf0bd8e73d21b82ce4fd 100644 Binary files a/ObservationSim/MockObject/__pycache__/Quasar.cpython-38.pyc and b/ObservationSim/MockObject/__pycache__/Quasar.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/__pycache__/SkybackgroundMap.cpython-38.pyc b/ObservationSim/MockObject/__pycache__/SkybackgroundMap.cpython-38.pyc index 65ab095f1b3e10cfad5ea1d0f851d05e3aa1c886..0a0d8a540d447b5836d3bd431ba41c24ce2d85c2 100644 Binary files a/ObservationSim/MockObject/__pycache__/SkybackgroundMap.cpython-38.pyc and b/ObservationSim/MockObject/__pycache__/SkybackgroundMap.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/__pycache__/Star.cpython-38.pyc b/ObservationSim/MockObject/__pycache__/Star.cpython-38.pyc index f44a7bcbebc2298c2ecca9ebde8a6e225b022472..99d9f2ca092f36cee6ce2c64491eafa0f64cbbf1 100644 Binary files a/ObservationSim/MockObject/__pycache__/Star.cpython-38.pyc and b/ObservationSim/MockObject/__pycache__/Star.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/__pycache__/__init__.cpython-38.pyc b/ObservationSim/MockObject/__pycache__/__init__.cpython-38.pyc index 554b40063422bdb8e43de5ae42e2865581c78285..5878d5ec01e34cf8dc426eb5e8394179de43efc2 100644 Binary files a/ObservationSim/MockObject/__pycache__/__init__.cpython-38.pyc and b/ObservationSim/MockObject/__pycache__/__init__.cpython-38.pyc differ diff --git a/ObservationSim/MockObject/__pycache__/_util.cpython-38.pyc b/ObservationSim/MockObject/__pycache__/_util.cpython-38.pyc index cb1a727ba9b7a2fd2a6e713ccfb8fc8263073960..96946b0149b4c2b5b41ca0e93be722f29c2855f5 100644 Binary files a/ObservationSim/MockObject/__pycache__/_util.cpython-38.pyc and b/ObservationSim/MockObject/__pycache__/_util.cpython-38.pyc differ diff --git a/ObservationSim/ObservationSim.py b/ObservationSim/ObservationSim.py index 2ad9e7c0c0eac5427d627e9f077cf95bf17534da..117e95ae82b22e6549d72fa9d09f15f16f0fa3b0 100755 --- a/ObservationSim/ObservationSim.py +++ b/ObservationSim/ObservationSim.py @@ -4,8 +4,8 @@ from Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip from MockObject import Catalog, MockObject, Star, Galaxy, Quasar, calculateSkyMap_split_g from PSF import PSFGauss, PSFInterp, FieldDistortion from _util import makeSubDir, getShearFiled, makeSubDir_PointingList -from astropy.time import Time as asTime from astropy.io import fits +from datetime import datetime import numpy as np import mpi4py.MPI as MPI import galsim @@ -47,7 +47,7 @@ class Observation(object): self.g1_field, self.g2_field, self.nshear = getShearFiled(config=self.config) - def runOneChip(self, chip, filt, chip_output, wcs_fp=None, psf_model=None, pointing_ID=0, ra_cen=None, dec_cen=None, img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None, cat_dir=None, sed_dir=None): + def runOneChip(self, chip, filt, chip_output, wcs_fp=None, psf_model=None, pointing_ID=0, ra_cen=None, dec_cen=None, img_rot=None, exptime=150., timestamp_obs=1621915200, pointing_type='MS', input_cat_name=None, shear_cat_file=None, cat_dir=None, sed_dir=None): if (ra_cen is None) or (dec_cen is None): ra_cen = self.config["ra_center"] @@ -80,169 +80,189 @@ class Observation(object): skyfile = os.path.join(self.path_dict["data_dir"], 'skybackground/sky_emiss_hubble_50_50_A.dat') sky_map = calculateSkyMap_split_g(xLen=chip.npix_x, yLen=chip.npix_y, blueLimit=filt.blue_limit, redLimit=filt.red_limit, skyfn=skyfile, conf=chip.sls_conf, pixelSize=chip.pix_scale, isAlongY=0) - # Load catalogues and templates - self.cat = Catalog(config=self.config, chip=chip, cat_dir=cat_dir, sed_dir=sed_dir, pRa=ra_cen, pDec=dec_cen, rotation=img_rot, template_dir=self.path_dict["template_dir"]) - self.nobj = len(self.cat.objs) - - # Loop over objects - missed_obj = 0 - bright_obj = 0 - dim_obj = 0 - for j in range(self.nobj): - # if j >= 20: - # break - obj = self.cat.objs[j] - - # Load SED - if obj.type == 'star': - normF = chip.normF_star - try: + if pointing_type == 'MS': + # Load catalogues and templates + self.cat = Catalog(config=self.config, chip=chip, cat_dir=cat_dir, sed_dir=sed_dir, pRa=ra_cen, pDec=dec_cen, rotation=img_rot, template_dir=self.path_dict["template_dir"]) + self.nobj = len(self.cat.objs) + + # Loop over objects + missed_obj = 0 + bright_obj = 0 + dim_obj = 0 + for j in range(self.nobj): + # if j >= 20: + # break + obj = self.cat.objs[j] + + # Load SED + if obj.type == 'star': + normF = chip.normF_star + try: + obj.load_SED( + survey_type=chip.survey_type, + normFilter=normF, + target_filt=filt, + sed_lib=self.cat.tempSED_star) + except Exception as e: + print(e) + continue + elif obj.type == 'galaxy': # or obj.type == quasar + normF = chip.normF_galaxy obj.load_SED( + sed_path=sed_dir, survey_type=chip.survey_type, + sed_templates=self.cat.tempSed_gal, normFilter=normF, - target_filt=filt, - sed_lib=self.cat.tempSED_star) - except Exception as e: - print(e) - continue - elif obj.type == 'galaxy': # or obj.type == quasar - normF = chip.normF_galaxy - obj.load_SED( - sed_path=sed_dir, - survey_type=chip.survey_type, - sed_templates=self.cat.tempSed_gal, - normFilter=normF, - target_filt=filt) - elif obj.type == 'quasar': - normF = chip.normF_galaxy - obj.load_SED( - sed_path=sed_dir, - survey_type=chip.survey_type, - sed_templates=self.cat.tempSed_gal, - normFilter=normF, - target_filt=filt) - - # Exclude very bright/dim objects (for now) - if filt.is_too_bright(mag=obj.getMagFilter(filt)): - # print("obj too birght!!", flush=True) - if obj.type != 'galaxy': - bright_obj += 1 + target_filt=filt) + elif obj.type == 'quasar': + normF = chip.normF_galaxy + obj.load_SED( + sed_path=sed_dir, + survey_type=chip.survey_type, + sed_templates=self.cat.tempSed_gal, + normFilter=normF, + target_filt=filt) + + # Exclude very bright/dim objects (for now) + if filt.is_too_bright(mag=obj.getMagFilter(filt)): + # print("obj too birght!!", flush=True) + if obj.type != 'galaxy': + bright_obj += 1 + obj.unload_SED() + continue + if filt.is_too_dim(mag=obj.getMagFilter(filt)): + # print("obj too dim!!", flush=True) + dim_obj += 1 obj.unload_SED() + # print(obj.getMagFilter(filt)) continue - if filt.is_too_dim(mag=obj.getMagFilter(filt)): - # print("obj too dim!!", flush=True) - dim_obj += 1 - obj.unload_SED() - # print(obj.getMagFilter(filt)) - continue - if self.config["shear_method"] == "constant": - if obj.type == 'star': - g1, g2 = 0, 0 - else: - g1, g2 = self.g1_field, self.g2_field - elif self.config["shear_method"] == "extra": - # TODO: every object with individual shear from input catalog(s) - g1, g2 = self.g1_field[j], self.g2_field[j] - - pos_img, offset, local_wcs = obj.getPosImg_Offset_WCS(img=chip.img, fdmodel=self.fd_model, chip=chip, verbose=False) - if pos_img.x == -1 or pos_img.y == -1: - # Exclude object which is outside the chip area (after field distortion) - # print("obj missed!!") - missed_obj += 1 - obj.unload_SED() - continue + if self.config["shear_method"] == "constant": + if obj.type == 'star': + g1, g2 = 0, 0 + else: + g1, g2 = self.g1_field, self.g2_field + elif self.config["shear_method"] == "extra": + # TODO: every object with individual shear from input catalog(s) + g1, g2 = self.g1_field[j], self.g2_field[j] + + pos_img, offset, local_wcs = obj.getPosImg_Offset_WCS(img=chip.img, fdmodel=self.fd_model, chip=chip, verbose=False) + if pos_img.x == -1 or pos_img.y == -1: + # Exclude object which is outside the chip area (after field distortion) + # print("obj missed!!") + missed_obj += 1 + obj.unload_SED() + continue - # Draw object & update output catalog - try: - if chip.survey_type == "photometric": - isUpdated, pos_shear = obj.drawObj_multiband( - tel=self.tel, - pos_img=pos_img, - psf_model=psf_model, - bandpass_list=filt.bandpass_sub_list, - filt=filt, - chip=chip, - g1=g1, - g2=g2, - exptime=exptime) - elif chip.survey_type == "spectroscopic": - isUpdated, pos_shear = obj.drawObj_slitless( - tel=self.tel, - pos_img=pos_img, - psf_model=psf_model, - bandpass_list=filt.bandpass_sub_list, - filt=filt, - chip=chip, - g1=g1, - g2=g2, - exptime=exptime, - normFilter=normF) - if isUpdated: - # TODO: add up stats - chip_output.cat_add_obj(obj, pos_img, pos_shear, g1, g2) + # Draw object & update output catalog + try: + if chip.survey_type == "photometric": + isUpdated, pos_shear = obj.drawObj_multiband( + tel=self.tel, + pos_img=pos_img, + psf_model=psf_model, + bandpass_list=filt.bandpass_sub_list, + filt=filt, + chip=chip, + g1=g1, + g2=g2, + exptime=exptime) + elif chip.survey_type == "spectroscopic": + isUpdated, pos_shear = obj.drawObj_slitless( + tel=self.tel, + pos_img=pos_img, + psf_model=psf_model, + bandpass_list=filt.bandpass_sub_list, + filt=filt, + chip=chip, + g1=g1, + g2=g2, + exptime=exptime, + normFilter=normF) + if isUpdated: + # TODO: add up stats + chip_output.cat_add_obj(obj, pos_img, pos_shear, g1, g2) + pass + else: + # print("object omitted", flush=True) + continue + except Exception as e: + print(e) pass - else: - # print("object omitted", flush=True) - continue - except Exception as e: - print(e) - pass - # Unload SED: - obj.unload_SED() - del obj + # Unload SED: + obj.unload_SED() + del obj - del psf_model - del self.cat + del psf_model + del self.cat print("check running:1: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing_ID, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True) # Detector Effects # =========================================================== chip.img = chip.addNoise(config=self.config, tel=self.tel, filt=filt, img=chip.img, sky_map=sky_map) - chip.img = chip.addEffects(config=self.config, img=chip.img, chip_output=chip_output, filt=filt, pointing_ID=pointing_ID) + # chip.img = chip.addEffects(config=self.config, img=chip.img, chip_output=chip_output, filt=filt, pointing_ID=pointing_ID) + + # whether to output zero, dark, flat calibration images. + chip.img = chip.addEffects( + config=self.config, + img=chip.img, + chip_output=chip_output, + filt=filt, + ra_cen=ra_cen, + dec_cen=dec_cen, + img_rot=img_rot, + pointing_ID=pointing_ID, + timestamp_obs=timestamp_obs, + pointing_type=pointing_type) - h_prim = generatePrimaryHeader( - xlen=chip.npix_x, - ylen=chip.npix_y, - pointNum = str(pointing_ID), - ra=ra_cen, - dec=dec_cen, - psize=chip.pix_scale, - row_num=chip.rowID, - col_num=chip.colID, - date=self.config["date_obs"], - time_obs=self.config["time_obs"], - im_type = 'MS') - h_ext = generateExtensionHeader( - xlen=chip.npix_x, - ylen=chip.npix_y, - ra=ra_cen, - dec=dec_cen, - pa=img_rot.deg, - gain=chip.gain, - readout=chip.read_noise, - dark=chip.dark_noise, - saturation=90000, - psize=chip.pix_scale, - row_num=chip.rowID, - col_num=chip.colID, - extName='SCI') - chip.img = galsim.Image(chip.img.array, dtype=np.uint16) - hdu1 = fits.PrimaryHDU(header=h_prim) - hdu2 = fits.ImageHDU(chip.img.array, header=h_ext) - hdu1 = fits.HDUList([hdu1, hdu2]) - fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits') - hdu1.writeto(fname, output_verify='ignore', overwrite=True) + if pointing_type == 'MS': + datetime_obs = datetime.fromtimestamp(timestamp_obs) + date_obs = datetime_obs.strftime("%y%m%d") + time_obs = datetime_obs.strftime("%H%M%S") + h_prim = generatePrimaryHeader( + xlen=chip.npix_x, + ylen=chip.npix_y, + pointNum = str(pointing_ID), + ra=ra_cen, + dec=dec_cen, + psize=chip.pix_scale, + row_num=chip.rowID, + col_num=chip.colID, + # date=self.config["date_obs"], + # time_obs=self.config["time_obs"], + date=date_obs, + time_obs=time_obs, + im_type='MS') + h_ext = generateExtensionHeader( + xlen=chip.npix_x, + ylen=chip.npix_y, + ra=ra_cen, + dec=dec_cen, + pa=img_rot.deg, + gain=chip.gain, + readout=chip.read_noise, + dark=chip.dark_noise, + saturation=90000, + psize=chip.pix_scale, + row_num=chip.rowID, + col_num=chip.colID, + extName='raw') + chip.img = galsim.Image(chip.img.array, dtype=np.uint16) + hdu1 = fits.PrimaryHDU(header=h_prim) + hdu2 = fits.ImageHDU(chip.img.array, header=h_ext) + hdu1 = fits.HDUList([hdu1, hdu2]) + fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits') + hdu1.writeto(fname, output_verify='ignore', overwrite=True) + print("# objects that are too bright %d out of %d"%(bright_obj, self.nobj)) + print("# objects that are too dim %d out of %d"%(dim_obj, self.nobj)) + print("# objects that are missed %d out of %d"%(missed_obj, self.nobj)) del chip.img print("check running:2: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing_ID, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True) - print("# objects that are too bright %d out of %d"%(bright_obj, self.nobj)) - print("# objects that are too dim %d out of %d"%(dim_obj, self.nobj)) - print("# objects that are missed %d out of %d"%(missed_obj, self.nobj)) - def runExposure(self, ra_cen=None, dec_cen=None, pointing_ID=0, img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None, oneChip=None): + def runExposure(self, ra_cen=None, dec_cen=None, timestamp_obs=1621915200, pointing_ID=0, pointing_type='MS', img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None, oneChip=None): if (ra_cen == None) or (dec_cen == None): ra_cen = self.config["ra_center"] @@ -269,6 +289,7 @@ class Observation(object): chip=chip, filt=filt, exptime=exptime, + pointing_type=pointing_type, pointing_ID=pointing_ID, subdir=sub_img_dir, prefix=prefix) @@ -282,11 +303,13 @@ class Observation(object): dec_cen=dec_cen, img_rot=img_rot, exptime=exptime, + timestamp_obs=timestamp_obs, + pointing_type=pointing_type, cat_dir=self.path_dict["cat_dir"], sed_dir=self.path_dict["SED_dir"]) print("finished running chip#%d..."%(chip.chipID), flush=True) - def runExposure_MPI_PointingList(self, ra_cen=None, dec_cen=None, pRange=None, img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None): + def runExposure_MPI_PointingList(self, ra_cen=None, dec_cen=None, pRange=None, timestamp_obs=[1621915200], pointing_type=['MS'], img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None): comm = MPI.COMM_WORLD ind_thread = comm.Get_rank() num_thread = comm.Get_size() @@ -302,6 +325,11 @@ class Observation(object): else: pStart = 0 + # TEMP + if len(timestamp_obs) == 1: + timestamp_obs *= len(pRange) + pointing_type *= len(pRange) + for ipoint in range(len(ra_cen)): for ichip in range(nchips_per_fp): i = ipoint*nchips_per_fp + ichip @@ -322,6 +350,7 @@ class Observation(object): chip=chip, filt=filt, exptime=exptime, + pointing_type=pointing_type[ipoint], pointing_ID=pointing_ID, subdir=sub_img_dir, prefix=prefix) @@ -333,7 +362,9 @@ class Observation(object): ra_cen=ra_cen[ipoint], dec_cen=dec_cen[ipoint], img_rot=img_rot, - exptime=exptime, + exptime=exptime, + timestamp_obs=timestamp_obs[ipoint], + pointing_type=pointing_type[ipoint], cat_dir=self.path_dict["cat_dir"], sed_dir=self.path_dict["SED_dir"]) print("finished running chip#%d..."%(chip.chipID), flush=True) \ No newline at end of file diff --git a/ObservationSim/PSF/PSFInterp/__pycache__/PSFConfig.cpython-38.pyc b/ObservationSim/PSF/PSFInterp/__pycache__/PSFConfig.cpython-38.pyc index b980756e5529c00079673e6685c0413509826f3c..8d69d156c3b0d307bddb9ac8cba6311623dedd24 100644 Binary files a/ObservationSim/PSF/PSFInterp/__pycache__/PSFConfig.cpython-38.pyc and b/ObservationSim/PSF/PSFInterp/__pycache__/PSFConfig.cpython-38.pyc differ diff --git a/ObservationSim/PSF/PSFInterp/__pycache__/PSFInterp.cpython-38.pyc b/ObservationSim/PSF/PSFInterp/__pycache__/PSFInterp.cpython-38.pyc index 4001ff5aa3edc5d1ded2fdfa824ac2a01f5b7f42..85d2ea33d95a0877e0ad35d1afd592187a96f0c2 100644 Binary files a/ObservationSim/PSF/PSFInterp/__pycache__/PSFInterp.cpython-38.pyc and b/ObservationSim/PSF/PSFInterp/__pycache__/PSFInterp.cpython-38.pyc differ diff --git a/ObservationSim/PSF/PSFInterp/__pycache__/PSFUtil.cpython-38.pyc b/ObservationSim/PSF/PSFInterp/__pycache__/PSFUtil.cpython-38.pyc index ea47204732df921c95837447a09a2ab3190c63ad..951c9b79a377319435febd7d00e7e311823d093f 100644 Binary files a/ObservationSim/PSF/PSFInterp/__pycache__/PSFUtil.cpython-38.pyc and b/ObservationSim/PSF/PSFInterp/__pycache__/PSFUtil.cpython-38.pyc differ diff --git a/ObservationSim/PSF/PSFInterp/__pycache__/__init__.cpython-38.pyc b/ObservationSim/PSF/PSFInterp/__pycache__/__init__.cpython-38.pyc index d362c497bcccdf2d4743f3fcc5e95ea70fdfc2b8..33d9ce0eebba8f744921b8afe5426de2e484f9f7 100644 Binary files a/ObservationSim/PSF/PSFInterp/__pycache__/__init__.cpython-38.pyc and b/ObservationSim/PSF/PSFInterp/__pycache__/__init__.cpython-38.pyc differ diff --git a/ObservationSim/PSF/__pycache__/FieldDistortion.cpython-38.pyc b/ObservationSim/PSF/__pycache__/FieldDistortion.cpython-38.pyc index 8ce0a9056673239a4b488e4f6266caa46218e8a4..7b3541b21799a858d05859591e7451af9418b63c 100644 Binary files a/ObservationSim/PSF/__pycache__/FieldDistortion.cpython-38.pyc and b/ObservationSim/PSF/__pycache__/FieldDistortion.cpython-38.pyc differ diff --git a/ObservationSim/PSF/__pycache__/PSFGauss.cpython-38.pyc b/ObservationSim/PSF/__pycache__/PSFGauss.cpython-38.pyc index ecbd67c31274875c2709616e6d33f4384d399ae7..6c27b983395a3d279b8df50c0fde4ae9b9d80183 100644 Binary files a/ObservationSim/PSF/__pycache__/PSFGauss.cpython-38.pyc and b/ObservationSim/PSF/__pycache__/PSFGauss.cpython-38.pyc differ diff --git a/ObservationSim/PSF/__pycache__/PSFModel.cpython-38.pyc b/ObservationSim/PSF/__pycache__/PSFModel.cpython-38.pyc index ec22b16c43ed9d28c2160ab605ed3eb4f6b5d51b..ed0556dc88d0372130fcc177bc5ff7ae411216c8 100644 Binary files a/ObservationSim/PSF/__pycache__/PSFModel.cpython-38.pyc and b/ObservationSim/PSF/__pycache__/PSFModel.cpython-38.pyc differ diff --git a/ObservationSim/PSF/__pycache__/__init__.cpython-38.pyc b/ObservationSim/PSF/__pycache__/__init__.cpython-38.pyc index afff9b789b503bef7517efb01aa773c7185a82b3..aa42e9e3346429a7b07e8737a26ba8df96761fa9 100644 Binary files a/ObservationSim/PSF/__pycache__/__init__.cpython-38.pyc and b/ObservationSim/PSF/__pycache__/__init__.cpython-38.pyc differ diff --git a/ObservationSim/Pointing.py b/ObservationSim/Pointing.py index c6b548d64b334712e7b090f069b8425b4a2afdd9..f38ebfb94f241f92166205faf059c50ca6cebcf4 100644 --- a/ObservationSim/Pointing.py +++ b/ObservationSim/Pointing.py @@ -1,8 +1,9 @@ import os import numpy as np from Config import ReadConfig +from datetime import datetime -work_dir = "/public/home/fangyuedong/CSST_new_framework/test/" +work_dir = "/public/home/fangyuedong/CSST/test/" data_dir = "/data/simudata/CSSOSDataProductsSims/data/" config_file = os.path.join(work_dir, "ObservationSim.cfg") @@ -25,5 +26,23 @@ f.close() pRA = np.array(pRA) pDEC = np.array(pDEC) +# Create calibration pointings +ncal = 1 # Define the number of calibration pointings +pointing_type = ['MS']*len(pRA) +pRA = np.append(pRA[:ncal], pRA) +pDEC = np.append(pDEC[:ncal], pDEC) +pointing_type = ['CAL']*ncal + pointing_type + +# Calculate starting time(s) +t0 = datetime(2021, 5, 25, 12, 0, 0) +t = datetime.timestamp(t0) +timestamp_obs = [] +for i in range(len(pointing_type)): + timestamp_obs.append(t) + if pointing_type[i] == 'CAL': + t += 3 * 5 * 60 # 3 calibration exposure + elif pointing_type[i] == 'MS': + t += 5 * 60 + # Define the range of pointing list -pRange = range(0, 3) \ No newline at end of file +pRange = range(0, 2) \ No newline at end of file diff --git a/ObservationSim/__pycache__/ObservationSim.cpython-38.pyc b/ObservationSim/__pycache__/ObservationSim.cpython-38.pyc index 05344adb1a60f85bd1043aa56ab31f02bcb72255..8927854296f4b745d0d52cb32028896a9b0f8857 100644 Binary files a/ObservationSim/__pycache__/ObservationSim.cpython-38.pyc and b/ObservationSim/__pycache__/ObservationSim.cpython-38.pyc differ diff --git a/ObservationSim/__pycache__/Pointing.cpython-38.pyc b/ObservationSim/__pycache__/Pointing.cpython-38.pyc index 36450eb7a6f8c4e1c4c4e5414c129bbdba2d90c6..57ecf0ddccba0c3b01b99904dbffed5c19a22a82 100644 Binary files a/ObservationSim/__pycache__/Pointing.cpython-38.pyc and b/ObservationSim/__pycache__/Pointing.cpython-38.pyc differ diff --git a/ObservationSim/__pycache__/_util.cpython-38.pyc b/ObservationSim/__pycache__/_util.cpython-38.pyc index 34a549fef6a08ae80088ececc7b3c74e9a680aac..1f0b66f8877f4d082bdf59a704fd98b443b9579e 100644 Binary files a/ObservationSim/__pycache__/_util.cpython-38.pyc and b/ObservationSim/__pycache__/_util.cpython-38.pyc differ diff --git a/ObservationSim/preprocess.py b/ObservationSim/preprocess.py index be712bdb63c132a261ee03e8cc17170a3d7e25fa..ee8ebf40dd8aa56b1a78b2b3dcc03c179483cd3a 100644 --- a/ObservationSim/preprocess.py +++ b/ObservationSim/preprocess.py @@ -7,7 +7,6 @@ imgDir = os.path.join(work_dir, config["mockImgDir"]) if not os.path.exists(imgDir): os.system("mkdir %s"%imgDir) -# prefix = "MSC_" + config["date_obs"] + "_" + config["time_obs"] + "_" prefix = "MSC_" for pointing_ID in range(len(pRA)): diff --git a/ObservationSim/run.pbs b/ObservationSim/run.pbs index f886da1f15f158c6c95077aa793e45d41b3fa80a..4502bc834f784b43984657ab8ad111e5bb5e4511 100755 --- a/ObservationSim/run.pbs +++ b/ObservationSim/run.pbs @@ -12,11 +12,11 @@ #PBS -q batch #PBS -u fangyuedong -NP=60 +NP=40 date echo $NP -# mpirun -np $NP --oversubscribe -H comput101 python /public/home/fangyuedong/CSST_new_framework/ObservationSim/runExposure.py +# mpirun -np $NP --oversubscribe -H comput101 python /public/home/fangyuedong/CSST/ObservationSim/runExposure.py -python /public/home/fangyuedong/CSST_new_framework/ObservationSim/preprocess.py -mpirun -np $NP python /public/home/fangyuedong/CSST_new_framework/ObservationSim/runExposure.py +python /public/home/fangyuedong/CSST/ObservationSim/preprocess.py +mpirun -np $NP python /public/home/fangyuedong/CSST/ObservationSim/runExposure.py diff --git a/ObservationSim/runExposure.py b/ObservationSim/runExposure.py index d56b5387addb57aa4115ca382512ecd092061fba..4f2eeff9a86c0b65600bed294ae8a0655f76c5d1 100755 --- a/ObservationSim/runExposure.py +++ b/ObservationSim/runExposure.py @@ -9,16 +9,15 @@ gc.enable() ############################################# # Testing run one exposure (NOT using MPI) -ipoint = 2 -pointRa = float('{:8.4f}'.format(pRA[ipoint])) -pointDec = float('{:8.4f}'.format(pDEC[ipoint])) +# ipoint = 2 +# pointRa = float('{:8.4f}'.format(pRA[ipoint])) +# pointDec = float('{:8.4f}'.format(pDEC[ipoint])) -cat_dir = 'point_RA' + str(pointRa) + '_DE' + str(pointDec) + '/' -obs = Observation(input_cat_dir=cat_dir, work_dir=work_dir, data_dir=data_dir) -obs = Observation(work_dir=work_dir, data_dir=data_dir) -obs.runExposure(ra_cen=pointRa, dec_cen=pointDec, pointing_ID=ipoint, oneChip=16) +# obs = Observation(work_dir=work_dir, data_dir=data_dir) +# obs.runExposure(ra_cen=pointRa, dec_cen=pointDec, pointing_ID=ipoint, oneChip=16) ############################################# # Testing run pointing list (using MPI) -# obs = Observation(work_dir=work_dir, data_dir=data_dir) -# obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange) +obs = Observation(work_dir=work_dir, data_dir=data_dir) +# obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type) +obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=150.) diff --git a/test/ObservationSim.cfg b/test/ObservationSim.cfg index 7865bc11aaf647a0d430ccded6c952b607c66cd2..ea2c2cef154274a0f95065d3362a12bd697cb43b 100755 --- a/test/ObservationSim.cfg +++ b/test/ObservationSim.cfg @@ -5,8 +5,8 @@ ################################################## # Observation setting -date_obs 210304 # Observation date [yymmdd] -time_obs 093000 # Observation time [hhmmss] +date_obs 210525 # Observation date [yymmdd] +time_obs 120000 # Observation time [hhmmss] ra_center 60.0 # Telesscope pointing center [degree] (Default) dec_center -40.0 psf_rcont 0.15,0.8 # Radius of the 80% flux concentration (for a Gaussian PSF) @@ -30,15 +30,16 @@ abs_back y # Y: add sky background + sky noise & dark # N: only add sky noise and dark noise add_bias y # Y/N: Whether to add bias-level to image shutter_effect y # Y/N: Whether to apply shutter effect -shutter_output y # Y/N: Whether to export shutter effect 16-bit image (<=65535) +shutter_output n # Y/N: Whether to export shutter effect 16-bit image (<=65535) flat_fielding y # Y/N: Whether to add flat-fielding effect bias_output y # Y/N: Whether to export bias frames dark_output y # Y/N: Whether to export the combined dark current file flat_output y # Y/N: Whether to export the combined flat-field file prnu_effect y # Y/N: Whether to apply PRNU effect -prnu_output y # Y/N: Whether to export the PRNU (pixel-to-pixel flat-fielding) file -non_linear y # Y/N: Whether to apply non-linearity +prnu_output n # Y/N: Whether to export the PRNU (pixel-to-pixel flat-fielding) file +non_linear n # Y/N: Whether to apply non-linearity cosmic_ray y # Y/N: Whether to add cosmic-ray +cray_differ y # Y/N: Different cosmic-ray for calibration files or not cte_trail y # Y/N: Whether to simulate CTE trails saturbloom y # Y/N: Whether to simulate Saturation and Blooming add_badcolumns y # Y/N: Whether to add bad columns @@ -50,9 +51,11 @@ df_strength 2.3 # Sillicon sensor diffusion strength bias_level 500 # e-/pixel gain 1.1 # Gain full_well 90000 # Full well level [e-] -NBias 10 # number of frames to be exported -NDark 10 # number of frames to be exported -NFlat 10 # number of frames to be exported +NBias 1 # Number of frames to be exported +NDark 1 # Number of frames to be exported +NFlat 1 # Number of frames to be exported +dark_exptime 300 # Exposure time for dark current frame +flat_exptime 150 # Exposure time for flat field frame #=========================================================== # Shear method