Commit b4f212dc authored by Zhang Xin's avatar Zhang Xin
Browse files

issue17:fix led model memory bug; issue19:header time is not utc bug;...

issue17:fix led model memory bug; issue19:header time is not utc bug; issue18:header data-obs\darktime\exposure time fix by shaoli's chart;add new star catalog:C6_50sqdeg_ns.py
parent 90a73995
This diff is collapsed.
......@@ -19,7 +19,7 @@ from astropy.time import Time
from astropy import wcs
from ObservationSim.Config._util import get_obs_id, get_file_type
from datetime import datetime
from datetime import datetime, timezone
# import socket
import platform
import toml
......@@ -352,7 +352,10 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointing_id = '00000001', po
# ccdnum = str(k)
datetime_obs = datetime.utcfromtimestamp(time_pt)
datetime_obs = datetime_obs.replace(tzinfo=timezone.utc)
# print(datetime_obs.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5])
datetime_obs = datetime.utcfromtimestamp(np.round(datetime_obs.timestamp(), 1))
# print(datetime_obs.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5])
# date_obs = datetime_obs.strftime("%y%m%d")
# time_obs = datetime_obs.strftime("%H%M%S%f")[:-5]
......@@ -551,30 +554,32 @@ def generateExtensionHeader(chip, xlen = 9216, ylen = 9232,ra = 60, dec = -40, p
h_ext['PIXSCAL1'] = pixel_scale
h_ext['PIXSCAL2'] = pixel_scale
h_ext['EXPTIME'] = exptime
h_ext['DARKTIME'] = exptime + readoutTime
h_ext['DARKTIME'] = exptime
datetime_obs = datetime.utcfromtimestamp(timestamp)
datetime_obs = datetime_obs.replace(tzinfo=timezone.utc)
tstart = Time(datetime_obs)
t_shutter_os = tstart
t_shutter_oe = Time(tstart.mjd + t_shutter_open / 86400., format="mjd")
t_shutter_co = Time(tstart.mjd + exptime / 86400., format="mjd")
t_shutter_ce = Time(tstart.mjd + (exptime + t_shutter_close) / 86400., format="mjd")
t_shutter_os1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_shutter_os.unix).timestamp(), 1))
t_shutter_os1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_shutter_os.unix).replace(tzinfo=timezone.utc).timestamp(), 1))
h_ext['SHTOPEN0'] = t_shutter_os1.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]
t_shutter_oe1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_shutter_oe.unix).timestamp(), 1))
t_shutter_oe1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_shutter_oe.unix).replace(tzinfo=timezone.utc).timestamp(), 1))
h_ext['SHTOPEN1'] = t_shutter_oe1.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]
t_shutter_co1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_shutter_co.unix).timestamp(), 1))
t_shutter_co1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_shutter_co.unix).replace(tzinfo=timezone.utc).timestamp(), 1))
h_ext['SHTCLOS0'] = t_shutter_co1.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]
t_shutter_ce1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_shutter_ce.unix).timestamp(), 1))
t_shutter_ce1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_shutter_ce.unix).replace(tzinfo=timezone.utc).timestamp(), 1))
h_ext['SHTCLOS1'] = t_shutter_ce1.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]
tstart_read = Time(tstart.mjd + exptime / 86400., format="mjd")
tend_read = Time(tstart.mjd + (exptime + readoutTime) / 86400., format="mjd")
# tstart1=tstart.datetime.replace(microsecond=round(tstart.datetime.microsecond, -5))
tstart1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(tstart_read.unix).timestamp(), 1))
tstart1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(tstart_read.unix).replace(tzinfo=timezone.utc).timestamp(), 1))
h_ext['ROTIME0'] = tstart1.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]
# tend_read1 = tend_read.datetime.replace(microsecond=round(tend_read.datetime.microsecond, -5))
tend_read1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(tend_read.unix).timestamp(), 1))
tend_read1 = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(tend_read.unix).replace(tzinfo=timezone.utc).timestamp(), 1))
h_ext['ROTIME1'] = tend_read1.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]
# h_ext['POS_ANG'] = pa
header_wcs = WCS_def(xlen=xlen, ylen=ylen, gapy=898.0, gapx1=534, gapx2=1309, ra_ref=ra, dec_ref=dec, pa=pa, pixel_scale=pixel_scale, pixel_size=pixel_size,
......
......@@ -56,6 +56,8 @@ class CatalogBase(metaclass=ABCMeta):
"teff":0.,
"logg":0.,
"feh":0.,
"DM":0.,
"stellarMass":1.,
# C6 galaxies parameters
"e1":0.,
"e2":0.,
......
......@@ -11,6 +11,7 @@ from scipy.interpolate import griddata
from astropy.table import Table
from ObservationSim.MockObject.SpecDisperser import SpecDisperser
from scipy import interpolate
import gc
from ObservationSim.MockObject.MockObject import MockObject
# from ObservationSim.Straylight import calculateSkyMap_split_g
......@@ -96,9 +97,48 @@ class FlatLED(object):
U = griddata(X_, Z_, (
M[self.chip.npix_y * i:self.chip.npix_y * (i + 1), self.chip.npix_x * j:self.chip.npix_x * (j + 1)],
N[self.chip.npix_y * i:self.chip.npix_y * (i + 1), self.chip.npix_x * j:self.chip.npix_x * (j + 1)]),
method='cubic')
method='linear')
U = U/np.mean(U)
flatImage = U*fluxLED[led_type]*1000
gc.collect()
return flatImage
###
### return LED flat, e/s
###
def getLEDImage1(self, led_type='LED1'):
# cwave = cwaves[led_type]
flat = fits.open(os.path.join(self.flatDir, 'model_' + cwaves_name[led_type] + 'nm.fits'))
xlen = flat[0].header['NAXIS1']
ylen = 601
i = self.chip.rowID - 1
j = self.chip.colID - 1
x = np.linspace(0, self.chip.npix_x, int(xlen/6.))
y = np.linspace(0, self.chip.npix_y, int(ylen/5.))
xx, yy = np.meshgrid(x, y)
a1 = flat[0].data[int(ylen*i/5.):int(ylen*i/5.)+int(ylen/5.), int(xlen*j/6.):int(xlen*j/6.)+int(xlen/6.)]
# z = np.sin((xx+yy+xx**2+yy**2))
# fInterp = interp2d(xx, yy, z, kind='linear')
X_ = np.hstack((xx.flatten()[:, None], yy.flatten()[:, None]))
Z_ = a1.flatten()
n_x = np.arange(0, self.chip.npix_x , 1)
n_y = np.arange(0, self.chip.npix_y, 1)
M, N = np.meshgrid(n_x, n_y)
U = griddata(X_, Z_, (
M[0:self.chip.npix_y, 0:self.chip.npix_x],
N[0:self.chip.npix_y, 0:self.chip.npix_x ]),
method='linear')
U = U/np.mean(U)
flatImage = U*fluxLED[led_type]*1000
gc.collect()
return flatImage
def drawObj_LEDFlat_img(self, led_type_list=['LED1'], exp_t_list=[0.1]):
......@@ -114,7 +154,9 @@ class FlatLED(object):
for i in np.arange(len(led_type_list)):
led_type = led_type_list[i]
exp_t = exp_t_list[i]
unitFlatImg = self.getLEDImage(led_type=led_type)
# unitFlatImg = self.getLEDImage(led_type=led_type)
unitFlatImg = self.getLEDImage1(led_type=led_type)
# print("---------------TEST mem:",np.mean(unitFlatImg))
led_wave = cwaves[led_type]
led_fwhm = cwaves_fwhm[led_type]
led_spec = self.gaussian1d_profile_led(led_wave, led_fwhm)
......@@ -134,6 +176,7 @@ class FlatLED(object):
ledStat = ledStat[0:int(led_type[3:])-1]+nledStat+ledStat[int(led_type[3:]):]
ledTimes[int(led_type[3:])-1] = exp_t * 1000
gc.collect()
return ledFlat, ledStat, ledTimes
def drawObj_LEDFlat_slitless(self, led_type_list=['LED1'], exp_t_list=[0.1]):
......@@ -150,7 +193,9 @@ class FlatLED(object):
for i in np.arange(len(led_type_list)):
led_type = led_type_list[i]
exp_t = exp_t_list[i]
unitFlatImg = self.getLEDImage(led_type=led_type)
# unitFlatImg = self.getLEDImage(led_type=led_type)
unitFlatImg = self.getLEDImage1(led_type=led_type)
# print("---------------TEST mem:",np.mean(unitFlatImg))
ledFlat_ = unitFlatImg*exp_t
ledFlat_ = ledFlat_ / mirro_eff[self.filt.filter_type]
ledFlat_.astype(np.float32)
......
......@@ -2,6 +2,11 @@ import numpy as np
from ObservationSim.MockObject import FlatLED
import galsim
from astropy.time import Time
from datetime import datetime, timezone
import gc
def add_LED_Flat(self, chip, filt, tel, pointing, catalog, obs_param):
if not hasattr(self, 'h_ext'):
......@@ -18,6 +23,7 @@ def add_LED_Flat(self, chip, filt, tel, pointing, catalog, obs_param):
self.updateHeaderInfo(header_flag='ext', keys = ['LEDSTAT'], values = [ledstat])
self.updateHeaderInfo(header_flag='ext', keys = ['LEDT01','LEDT02','LEDT03','LEDT04','LEDT05','LEDT06','LEDT07','LEDT08','LEDT09','LEDT10','LEDT11','LEDT12','LEDT13','LEDT14'], values = letts)
if obs_param["shutter_effect"] == True:
pf_map = pf_map * chip.shutter_img
pf_map = np.array(pf_map, dtype='float32')
......@@ -26,4 +32,20 @@ def add_LED_Flat(self, chip, filt, tel, pointing, catalog, obs_param):
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [False,'','','',''])
chip.img = chip.img + pf_map
# renew header info
datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
datetime_obs = datetime_obs.replace(tzinfo=timezone.utc)
t_obs = Time(datetime_obs)
##ccd刷新2s,等待0.5s,开灯后等待0.5s,开始曝光
t_obs_renew = Time(t_obs.mjd - (2.+0.5 +0.5) / 86400., format="mjd")
t_obs_utc = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_obs_renew.unix).replace(tzinfo=timezone.utc).timestamp(), 1))
self.updateHeaderInfo(header_flag='prim', keys = ['DATE-OBS'], values = [t_obs_utc.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]])
#dark time : 曝光时间+刷新后等带时间0.5s+点亮灯后0.5s+关闭快门时间1.5s+管快门后关灯前0.5+关灯后读出前等待0.5s
self.updateHeaderInfo(header_flag='ext', keys = ['DARKTIME'], values = [0.5+0.5+1.5+0.5+0.5+pointing.exp_time])
gc.collect()
return chip, filt, tel, pointing
\ No newline at end of file
......@@ -7,6 +7,9 @@ import galsim
from ObservationSim._util import get_shear_field
from ObservationSim.PSF import PSFGauss, FieldDistortion, PSFInterp, PSFInterpSLS
from astropy.time import Time
from datetime import datetime, timezone
def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
# Get exposure time
......@@ -59,6 +62,7 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
# Get chip WCS
if not hasattr(self, 'h_ext'):
_, _ = self.prepare_headers(chip=chip, pointing=pointing)
chip_wcs = galsim.FitsWCS(header = self.h_ext)
# Loop over objects
......@@ -211,4 +215,19 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
chip.img *= flat_normal
del flat_normal
# renew header info
datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
datetime_obs = datetime_obs.replace(tzinfo=timezone.utc)
t_obs = Time(datetime_obs)
##ccd刷新2s,等待0.5s,开始曝光
t_obs_renew = Time(t_obs.mjd - (2.+0.5) / 86400., format="mjd")
t_obs_utc = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_obs_renew.unix).replace(tzinfo=timezone.utc).timestamp(), 1))
self.updateHeaderInfo(header_flag='prim', keys = ['DATE-OBS'], values = [t_obs_utc.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]])
#dark time : 曝光时间+刷新后等带时间0.5s+关闭快门时间1.5s+管快门后读出前等待0.5s
self.updateHeaderInfo(header_flag='ext', keys = ['DARKTIME'], values = [0.5+1.5+0.5+pointing.exp_time])
return chip, filt, tel, pointing
\ No newline at end of file
......@@ -3,6 +3,9 @@ import galsim
from ObservationSim.Straylight import calculateSkyMap_split_g
from ObservationSim.Instrument import FilterParam
from astropy.time import Time
from datetime import datetime, timezone
def add_sky_background_sci(self, chip, filt, tel, pointing, catalog, obs_param):
# Get exposure time
......@@ -120,4 +123,20 @@ def add_sky_background(self, chip, filt, tel, pointing, catalog, obs_param):
chip, filt, tel, pointing = self.add_sky_background_sci(chip, filt, tel, pointing, catalog, obs_param)
else:
chip, filt, tel, pointing = self.add_sky_flat_calibration(chip, filt, tel, pointing, catalog, obs_param)
# renew header info
datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
datetime_obs = datetime_obs.replace(tzinfo=timezone.utc)
t_obs = Time(datetime_obs)
##ccd刷新2s,等待0.5s,开始曝光
t_obs_renew = Time(t_obs.mjd - (2.+0.5) / 86400., format="mjd")
t_obs_utc = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_obs_renew.unix).replace(tzinfo=timezone.utc).timestamp(), 1))
self.updateHeaderInfo(header_flag='prim', keys = ['DATE-OBS'], values = [t_obs_utc.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]])
#dark time : 曝光时间+刷新后等带时间0.5s+关闭快门时间1.5s+管快门后读出前等待0.5s
self.updateHeaderInfo(header_flag='ext', keys = ['DARKTIME'], values = [0.5+1.5+0.5+pointing.exp_time])
return chip, filt, tel, pointing
\ No newline at end of file
......@@ -5,6 +5,9 @@ from astropy.io import fits
from ObservationSim.Instrument.Chip import ChipUtils as chip_utils
from ObservationSim.Instrument.Chip import Effects
from astropy.time import Time
from datetime import datetime, timezone
def add_prescan_overscan(self, chip, filt, tel, pointing, catalog, obs_param):
self.chip_output.Log_info("Apply pre/over-scan")
chip.img = chip_utils.AddPreScan(GSImage=chip.img,
......@@ -42,7 +45,17 @@ def quantization_and_output(self, chip, filt, tel, pointing, catalog, obs_param)
if not hasattr(self, 'h_ext'):
_, _ = self.prepare_headers(chip=chip, pointing=pointing)
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [False,'','','',''])
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1','EXPTIME'], values = [False,'','','','',0.0])
# renew header info
datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
datetime_obs = datetime_obs.replace(tzinfo=timezone.utc)
t_obs = Time(datetime_obs)
##ccd刷新2s,等待0.5s,开灯后等待0.5s,开始曝光
t_obs_renew = Time(t_obs.mjd - 2. / 86400., format="mjd")
t_obs_utc = datetime.utcfromtimestamp(np.round(datetime.utcfromtimestamp(t_obs_renew.unix).replace(tzinfo=timezone.utc).timestamp(), 1))
self.updateHeaderInfo(header_flag='prim', keys = ['DATE-OBS'], values = [t_obs_utc.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-5]])
gains1 = list(chip.gain_channel[0:8])
gains2 = list(chip.gain_channel[8:])
......
---
###############################################
#
# Configuration file for CSST simulation
# Overall settings
# CSST-Sim Group, 2024/01/08
#
###############################################
# Base diretories and naming setup
# can add some of the command-line arguments here as well;
# ok to pass either way or both, as long as they are consistent
work_dir: "/home/zhangxin/CSST_SIM/CSST_sim_scheduler/"
run_name: "QSO_50sqdeg_test"
# Project cycle and run counter are used to name the outputs
project_cycle: 9
run_counter: 1
# Run options
run_option:
use_mpi: NO
# Output catalog only?
# If yes, no imaging simulation will be run. Only the catalogs
# of corresponding footprints will be generated.
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure the input catalog: options should be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "/nfsdata/share/CSSOSDataProductsSims/data/Catalog_C6_20221212/"
star_cat: "star_catalog/"
# galaxy_cat: "qsocat/cat2CSSTSim_bundle-50sqDeg/"
SED_templates_path:
star_SED: "/nfsdata/share/CSSOSDataProductsSims/data/Catalog_C6_20221212/star_catalog/"
# galaxy_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/sedlibs/"
# AGN_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/qsocat/qsosed/"
# Only simulate stars?
star_only: YES
# Only simulate galaxies?
galaxy_only: NO
###############################################
# Observation setting
###############################################
obs_setting:
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_file: "/nfsdata/share/CSSOSDataProductsSims/data/pointing_50_1_n.dat"
obs_config_file: "/home/zhangxin/CSST_SIM/CSST_sim_scheduler/csst-simulation/config/obs_config_SCI_WIDE_phot.yaml"
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [1]
# Whether to enable astrometric modeling
enable_astrometric_model: NO
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
# mag_sat_margin: -15.
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
# psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
# PSF models for photometry survey simulation
psf_pho_dir: "/nfsdata/share/CSSOSDataProductsSims/data/psfCube1"
# PSF models for slitless spectrum survey simulation
psf_sls_dir: "/nfsdata/share/CSSOSDataProductsSims/data/SLS_PSF_PCA_fp/"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": get shear values from catalog
shear_type: "constant"
# For constant shear field
reduced_g1: 0.
reduced_g2: 0.
###############################################
# Output options
###############################################
output_setting:
output_format: "channels" # Whether to export as 16 channels (subimages) with pre- and over-scan ("image"/"channels")
shutter_output: NO # Whether to export shutter effect 16-bit image
prnu_output: NO # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
......@@ -80,7 +80,7 @@ setup(name='CSSTSim',
'ObservationSim.Instrument.data.throughputs': ['*.txt', '*.dat'],
'ObservationSim.Instrument.data.sls_conf': ['*.conf', '*.fits'],
'ObservationSim.Instrument.data.flatCube': ['*.fits'],
'Catalog.data': ['*.fits'],
'Catalog.data': ['*.fits','*.so'],
'ObservationSim.Config.Header':['*.header','*.lst'],
'ObservationSim.Straylight.data': ['*.dat'],
'ObservationSim.Straylight.data.sky': ['*.dat'],
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment