Skip to content
GitLab
Commit 8f6e5f8f authored by Fang Yuedong's avatar Fang Yuedong
Browse files

overall and obs configs for SCIE observation

parent 633fbbab
Hide whitespace changes
Inline Side-by-side
ObservationSim/Config/ChipOutput.py
View file @ 8f6e5f8f
......@@ -60,7 +60,7 @@ class ChipOutput(object):
self.hdr += additional_column_names
def create_output_file(self):
if self.pointing_type == 'SCI':
if self.pointing_type == 'SCIE':
self.cat = open(os.path.join(self.subdir, self.cat_name), "w")
self.logger.info("Creating catalog file %s ...\n"%(os.path.join(self.subdir, self.cat_name)))
if not self.hdr.endswith("\n"):
......@@ -68,8 +68,6 @@ class ChipOutput(object):
self.cat.write(self.hdr)
def cat_add_obj(self, obj, pos_img, pos_shear):
# ximg = pos_img.x - self.chip.bound.xmin + 1.0
# yimg = pos_img.y - self.chip.bound.ymin + 1.0
# self.logger.info('-------------debug-----------------')
# self.logger.info('from global',ximg, yimg)
ximg = obj.real_pos.x + 1.0
......
ObservationSim/Config/Config.py
View file @ 8f6e5f8f
......@@ -19,7 +19,7 @@ def config_dir(config, work_dir=None, data_dir=None):
path_dict["data_dir"] = data_dir
# PSF data directory
if config["psf_setting"]["psf_model"] == "Interp":
path_dict["psf_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_dir"])
path_dict["psf_pho_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_pho_dir"])
path_dict["psf_sls_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_sls_dir"])
return path_dict
......
ObservationSim/Config/Pointing.py
View file @ 8f6e5f8f
......@@ -71,12 +71,13 @@ class Pointing(object):
self.survey_field_type = "DEEP"
# Load the configuration file for this particular pointing
# [TODO]
self.obs_config_file = "/share/home/fangyuedong/20231211/csst-simulation/config/obs_config_SCI_WIDE_phot.yaml"
with open(self.obs_config_file, "r") as stream:
try:
self.obs_param = yaml.safe_load(stream)
except yaml.YAMLError as exc:
print(exc)
self.pointing_type = self.obs_config_file["obs_type"]
self.pointing_type = self.obs_param["obs_type"]
else:
self.timestamp = t
ObservationSim/Instrument/Chip/Chip.py
View file @ 8f6e5f8f
......@@ -57,32 +57,27 @@ class Chip(FocalPlane):
for key in chip_dict:
setattr(self, key, chip_dict[key])
self.fdModel = None
if self.filter_type == "FGS":
if ("field_dist" in config) and (config["ins_effects"]["field_dist"]) == False:
self.fdModel = None
else:
fgs_name = self.chip_name[0:4]
try:
with pkg_resources.files('ObservationSim.Instrument.data.field_distortion').joinpath("FieldDistModelGlobal_pr4_%s.pickle"%(fgs_name.lower())) as field_distortion:
with open(field_distortion, "rb") as f:
self.fdModel = pickle.load(f)
except AttributeError:
with pkg_resources.path('ObservationSim.Instrument.data.field_distortion', "FieldDistModelGlobal_pr4_%s.pickle"%(fgs_name.lower())) as field_distortion:
with open(field_distortion, "rb") as f:
self.fdModel = pickle.load(f)
fgs_name = self.chip_name[0:4]
try:
with pkg_resources.files('ObservationSim.Instrument.data.field_distortion').joinpath("FieldDistModelGlobal_pr4_%s.pickle"%(fgs_name.lower())) as field_distortion:
with open(field_distortion, "rb") as f:
self.fdModel = pickle.load(f)
except AttributeError:
with pkg_resources.path('ObservationSim.Instrument.data.field_distortion', "FieldDistModelGlobal_pr4_%s.pickle"%(fgs_name.lower())) as field_distortion:
with open(field_distortion, "rb") as f:
self.fdModel = pickle.load(f)
else:
# Get the corresponding field distortion model
if ("field_dist" in config) and (config["ins_effects"]["field_dist"] == False):
self.fdModel = None
else:
try:
with pkg_resources.files('ObservationSim.Instrument.data.field_distortion').joinpath("FieldDistModel_v2.0.pickle") as field_distortion:
with open(field_distortion, "rb") as f:
self.fdModel = pickle.load(f)
except AttributeError:
with pkg_resources.path('ObservationSim.Instrument.data.field_distortion', "FieldDistModelGlobal_mainFP_v1.0.pickle") as field_distortion:
with open(field_distortion, "rb") as f:
self.fdModel = pickle.load(f)
try:
with pkg_resources.files('ObservationSim.Instrument.data.field_distortion').joinpath("FieldDistModel_v2.0.pickle") as field_distortion:
with open(field_distortion, "rb") as f:
self.fdModel = pickle.load(f)
except AttributeError:
with pkg_resources.path('ObservationSim.Instrument.data.field_distortion', "FieldDistModelGlobal_mainFP_v1.0.pickle") as field_distortion:
with open(field_distortion, "rb") as f:
self.fdModel = pickle.load(f)
# Get boundary (in pix)
self.bound = self.getChipLim()
......@@ -117,11 +112,8 @@ class Chip(FocalPlane):
self.effCurve = self._getChipEffCurve(self.filter_type)
self._getCRdata()
# Define the sensor model
if "bright_fatter" in config["ins_effects"] and config["ins_effects"]["bright_fatter"] == True and self.survey_type == "photometric":
self.sensor = galsim.SiliconSensor(strength=self.df_strength, treering_func=treering_func)
else:
self.sensor = galsim.Sensor()
# # Define the sensor model
self.sensor = galsim.Sensor()
self.flat_cube = None # for spectroscopic flat field cube simulation
......@@ -132,9 +124,9 @@ class Chip(FocalPlane):
self.rotate_angle = 0.
self.overscan = 1000
# Override default values
for key in ["gain", "bias_level, dark_exptime", "flat_exptime", "readout_time", "full_well", "read_noise", "dark_noise", "overscan"]:
if key in config["ins_effects"]:
setattr(self, key, config["ins_effects"][key])
# for key in ["gain", "bias_level, dark_exptime", "flat_exptime", "readout_time", "full_well", "read_noise", "dark_noise", "overscan"]:
# if key in config["ins_effects"]:
# setattr(self, key, config["ins_effects"][key])
def _getChipRowCol(self):
self.rowID, self.colID = self.getChipRowCol(self.chipID)
......
ObservationSim/MockObject/Galaxy.py
View file @ 8f6e5f8f
import numpy as np
import galsim
import os, sys
import astropy.constants as cons
from astropy.table import Table
from scipy import interpolate
from ObservationSim.MockObject._util import eObs, integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG,convolveGaussXorders
from ObservationSim.MockObject._util import eObs, integrate_sed_bandpass, getNormFactorForSpecWithABMAG
from ObservationSim.MockObject.SpecDisperser import SpecDisperser
from ObservationSim.MockObject.MockObject import MockObject
......@@ -64,7 +61,12 @@ class Galaxy(MockObject):
bulge_shape = galsim.Shear(g1=self.e1_bulge, g2=self.e2_bulge)
bulge = bulge.shear(bulge_shape)
gal = self.bfrac * bulge + (1.0 - self.bfrac) * disk
if self.bfrac == 0:
gal = disk
elif self.bfrac == 1:
gal = bulge
else:
gal = self.bfrac * bulge + (1.0 - self.bfrac) * disk
gal = gal.withFlux(nphotons)
if fd_shear is not None:
g1 += fd_shear.g1
......@@ -155,9 +157,13 @@ class Galaxy(MockObject):
# Get PSF model
psf, pos_shear = psf_model.get_PSF(chip=chip, pos_img=pos_img, bandpass=bandpass, folding_threshold=folding_threshold)
gal_temp = self.bfrac * bulge + (1.0 - self.bfrac) * disk
if self.bfrac == 0:
gal_temp = disk
elif self.bfrac == 1:
gal_temp = bulge
else:
gal_temp = self.bfrac * bulge + (1.0 - self.bfrac) * disk
gal_temp = gal_temp.shear(gal_shear)
gal_temp = gal_temp.withFlux(nphotons)
if not big_galaxy: # Not apply PSF for very big galaxy
gal_temp = galsim.Convolve(psf, gal_temp)
......@@ -172,14 +178,6 @@ class Galaxy(MockObject):
# kfrac = np.random.random()*(1.0 - self.bfrac)
# gal = self.bfrac * bulge + (1.0 - self.bfrac - kfrac) * disk + kfrac * knots
# # [C6 TEST]
# print('xmax = %d, ymax = %d '%(xmax, ymax))
# # Output memory usage
# snapshot = tracemalloc.take_snapshot()
# top_stats = snapshot.statistics('lineno')
# for stat in top_stats[:10]:
# print(stat)
# stamp = gal.drawImage(wcs=chip_wcs_local, method='phot', offset=offset, save_photons=True)
stamp = gal.drawImage(wcs=chip_wcs_local, offset=offset)
if np.sum(np.isnan(stamp.array)) > 0:
......
ObservationSim/ObservationSim.py
View file @ 8f6e5f8f
......@@ -4,22 +4,16 @@ import mpi4py.MPI as MPI
import galsim
import psutil
import gc
from astropy.io import fits
from datetime import datetime
import traceback
from ObservationSim.Config import config_dir, ChipOutput
from ObservationSim.Config.Header import generatePrimaryHeader, generateExtensionHeader
from ObservationSim.Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip
from ObservationSim.Instrument.Chip import Effects
from ObservationSim.Instrument.Chip import ChipUtils as chip_utils
from ObservationSim.Straylight import calculateSkyMap_split_g
from ObservationSim.PSF import PSFGauss, FieldDistortion, PSFInterp, PSFInterpSLS
from ObservationSim._util import get_shear_field, makeSubDir_PointingList
from ObservationSim._util import makeSubDir_PointingList
from ObservationSim.Astrometry.Astrometry_util import on_orbit_obs_position
from ObservationSim.MockObject import FlatLED
from ObservationSim.SimSteps import SimSteps, SIM_STEP_TYPES
class Observation(object):
......@@ -30,7 +24,7 @@ class Observation(object):
self.filter_param = FilterParam()
self.Catalog = Catalog
def prepare_chip_for_exposure(self, chip, ra_cen, dec_cen, pointing):
def prepare_chip_for_exposure(self, chip, ra_cen, dec_cen, pointing, wcs_fp=None):
# Get WCS for the focal plane
if wcs_fp == None:
wcs_fp = self.focal_plane.getTanWCS(ra_cen, dec_cen, pointing.img_pa, chip.pix_scale)
......@@ -45,8 +39,8 @@ class Observation(object):
seed=int(self.config["random_seeds"]["seed_poisson"]) + pointing.id*30 + chip.chipID, sky_level=0.)
# Get flat, shutter, and PRNU images
_, chip.flat_normal = chip_utils.get_flat(img=chip.img, seed=int(self.config["random_seeds"]["seed_flat"]))
chip.shuttimg = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735, dt=1E-3)
chip.flat_img, _ = chip_utils.get_flat(img=chip.img, seed=int(self.config["random_seeds"]["seed_flat"]))
chip.shutter_img = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735, dt=1E-3)
chip.prnu_img = Effects.PRNU_Img(xsize=chip.npix_x, ysize=chip.npix_y, sigma=0.01,
seed=int(self.config["random_seeds"]["seed_prnu"]+chip.chipID))
......@@ -92,14 +86,7 @@ class Observation(object):
ra_cen = pointing.ra
dec_cen = pointing.dec
# # Get WCS for the focal plane
# if wcs_fp == None:
# wcs_fp = self.focal_plane.getTanWCS(ra_cen, dec_cen, pointing.img_pa, chip.pix_scale)
# # Create chip Image
# chip.img = galsim.ImageF(chip.npix_x, chip.npix_y)
# chip.img.setOrigin(chip.bound.xmin, chip.bound.ymin)
# chip.img.wcs = wcs_fp
# Prepare necessary chip properties for simulation
chip = self.prepare_chip_for_exposure(chip, ra_cen, dec_cen, pointing)
# Load catalogues
......@@ -109,6 +96,7 @@ class Observation(object):
sim_steps = SimSteps(overall_config=self.config, chip_output=chip_output, all_filters=self.all_filters)
for step in pointing.obs_param["call_sequence"]:
chip_output.Log_info("Starting simulation step: %s, calling function: %s"%(step, SIM_STEP_TYPES[step]))
obs_param = pointing.obs_param["call_sequence"][step]
step_name = SIM_STEP_TYPES[step]
try:
......@@ -116,463 +104,151 @@ class Observation(object):
chip, filt, tel, pointing = step_func(
chip=chip,
filt=filt,
tel=tel,
tel=self.tel,
pointing=pointing,
catalog=self.cat,
obs_param=obs_param)
chip_output.Log_info("Finished simulation step: %s"%(step))
except Exception as e:
traceback.print_exc()
chip_output.Log_error(e)
continue
# if self.config["obs_setting"]["enable_straylight_model"]:
# filt.setFilterStrayLightPixel(jtime = pointing.jdt, sat_pos = np.array([pointing.sat_x, pointing.sat_y, pointing.sat_z]), pointing_radec = np.array([pointing.ra,pointing.dec]), sun_pos = np.array([pointing.sun_x,pointing.sun_y,pointing.sun_z]))
# chip_output.Log_info("========================sky pix========================")
# chip_output.Log_info(filt.sky_background)
# if chip.survey_type == "photometric":
# sky_map = None
# elif chip.survey_type == "spectroscopic":
# # chip.loadSLSFLATCUBE(flat_fn='flat_cube.fits')
# flat_normal = np.ones_like(chip.img.array)
# if self.config["ins_effects"]["flat_fielding"] == True:
# chip_output.Log_info("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding"%chip.chipID)
# msg = str(chip.img.bounds)
# chip_output.Log_info(msg)
# flat_img = Effects.MakeFlatSmooth(
# chip.img.bounds,
# int(self.config["random_seeds"]["seed_flat"]))
# flat_normal = flat_normal * flat_img.array / np.mean(flat_img.array)
# if self.config["ins_effects"]["shutter_effect"] == True:
# chip_output.Log_info("SLS flat preprocess,CHIP %d : Apply shutter effect"%chip.chipID)
# shuttimg = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735,
# dt=1E-3) # shutter effect normalized image for this chip
# flat_normal = flat_normal*shuttimg
# flat_normal = np.array(flat_normal,dtype='float32')
# sky_map = calculateSkyMap_split_g(
# skyMap=flat_normal,
# blueLimit=filt.blue_limit,
# redLimit=filt.red_limit,
# conf=chip.sls_conf,
# pixelSize=chip.pix_scale,
# isAlongY=0,
# flat_cube=chip.flat_cube, zoldial_spec = filt.zodical_spec)
# sky_map = sky_map+filt.sky_background
# del flat_normal
# if pointing.pointing_type == 'SCI':
# # Load catalogues and templates
# self.cat = self.Catalog(config=self.config, chip=chip, pointing=pointing, cat_dir=cat_dir, sed_dir=sed_dir, chip_output=chip_output, filt=filt)
# chip_output.create_output_file()
# self.nobj = len(self.cat.objs)
# for ifilt in range(len(self.all_filter)):
# temp_filter = self.all_filter[ifilt]
# # Update the limiting magnitude using exposure time in pointing
# temp_filter.update_limit_saturation_mags(exptime=pointing.get_full_depth_exptime(temp_filter.filter_type), chip=chip)
# # Select cutting band filter for saturation/limiting magnitude
# if temp_filter.filter_type.lower() == self.config["obs_setting"]["cut_in_band"].lower():
# cut_filter = temp_filter
# if self.config["ins_effects"]["field_dist"] == True:
# self.fd_model = FieldDistortion(chip=chip, img_rot=pointing.img_pa.deg)
# else:
# self.fd_model = None
# # Loop over objects
# missed_obj = 0
# bright_obj = 0
# dim_obj = 0
# h_ext = generateExtensionHeader(
# chip=chip,
# xlen=chip.npix_x,
# ylen=chip.npix_y,
# ra=pointing.ra,
# dec=pointing.dec,
# pa=pointing.img_pa.deg,
# gain=chip.gain,
# readout=chip.read_noise,
# dark=chip.dark_noise,
# saturation=90000,
# pixel_scale=chip.pix_scale,
# pixel_size=chip.pix_size,
# xcen=chip.x_cen,
# ycen=chip.y_cen,
# extName='SCI',
# timestamp = pointing.timestamp,
# exptime = pointing.exp_time,
# readoutTime = chip.readout_time)
# chip_wcs = galsim.FitsWCS(header=h_ext)
# for j in range(self.nobj):
# # # (DEBUG)
# # if j >= 10:
# # break
# obj = self.cat.objs[j]
# # (DEBUG)
# # if obj.getMagFilter(filt)>20:
# # continue
# # load and convert SED; also caculate object's magnitude in all CSST bands
# try:
# sed_data = self.cat.load_sed(obj)
# norm_filt = self.cat.load_norm_filt(obj)
# obj.sed, obj.param["mag_%s"%filt.filter_type.lower()], obj.param["flux_%s"%filt.filter_type.lower()] = self.cat.convert_sed(
# mag=obj.param["mag_use_normal"],
# sed=sed_data,
# target_filt=filt,
# norm_filt=norm_filt,
# )
# _, obj.param["mag_%s"%cut_filter.filter_type.lower()], obj.param["flux_%s"%cut_filter.filter_type.lower()] = self.cat.convert_sed(
# mag=obj.param["mag_use_normal"],
# sed=sed_data,
# target_filt=cut_filter,
# norm_filt=norm_filt,
# )
# except Exception as e:
# traceback.print_exc()
# chip_output.Log_error(e)
# continue
# # [TODO] Testing
# # chip_output.Log_info("mag_%s = %.3f"%(filt.filter_type.lower(), obj.param["mag_%s"%filt.filter_type.lower()]))
# # Exclude very bright/dim objects (for now)
# if cut_filter.is_too_bright(
# mag=obj.param["mag_%s"%self.config["obs_setting"]["cut_in_band"].lower()],
# margin=self.config["obs_setting"]["mag_sat_margin"]):
# chip_output.Log_info("obj %s too birght!! mag_%s = %.3f"%(obj.id, cut_filter.filter_type, obj.param["mag_%s"%self.config["obs_setting"]["cut_in_band"].lower()]))
# bright_obj += 1
# obj.unload_SED()
# continue
# if filt.is_too_dim(
# mag=obj.getMagFilter(filt),
# margin=self.config["obs_setting"]["mag_lim_margin"]):
# chip_output.Log_info("obj %s too dim!! mag_%s = %.3f"%(obj.id, filt.filter_type, obj.getMagFilter(filt)))
# dim_obj += 1
# obj.unload_SED()
# continue
# # Get corresponding shear values
# if self.config["shear_setting"]["shear_type"] == "constant":
# if obj.type == 'star':
# obj.g1, obj.g2 = 0., 0.
# else:
# obj.g1, obj.g2 = self.g1_field, self.g2_field
# elif self.config["shear_setting"]["shear_type"] == "catalog":
# pass
# else:
# chip_output.Log_error("Unknown shear input")
# raise ValueError("Unknown shear input")
# # Get position of object on the focal plane
# pos_img, offset, local_wcs, real_wcs, fd_shear = obj.getPosImg_Offset_WCS(img=chip.img, fdmodel=self.fd_model, chip=chip, verbose=False, chip_wcs=chip_wcs, img_header=h_ext)
# # [TODO] For now, only consider objects which their centers (after field distortion) are projected within the focal plane
# # Otherwise they will be considered missed objects
# # if pos_img.x == -1 or pos_img.y == -1 or (not chip.isContainObj(x_image=pos_img.x, y_image=pos_img.y, margin=0.)):
# if pos_img.x == -1 or pos_img.y == -1:
# chip_output.Log_info('obj_ra = %.6f, obj_dec = %.6f, obj_ra_orig = %.6f, obj_dec_orig = %.6f'%(obj.ra, obj.dec, obj.ra_orig, obj.dec_orig))
# chip_output.Log_error("Objected missed: %s"%(obj.id))
# missed_obj += 1
# obj.unload_SED()
# continue
# # Draw object & update output catalog
# try:
# if self.config["run_option"]["out_cat_only"]:
# isUpdated = True
# obj.real_pos = obj.getRealPos(chip.img, global_x=obj.posImg.x, global_y=obj.posImg.y, img_real_wcs=obj.chip_wcs)
# pos_shear = 0.
# elif chip.survey_type == "photometric" and not self.config["run_option"]["out_cat_only"]:
# isUpdated, pos_shear = obj.drawObj_multiband(
# tel=self.tel,
# pos_img=pos_img,
# psf_model=psf_model,
# bandpass_list=filt.bandpass_sub_list,
# filt=filt,
# chip=chip,
# g1=obj.g1,
# g2=obj.g2,
# exptime=pointing.exp_time,
# fd_shear=fd_shear)
# elif chip.survey_type == "spectroscopic" and not self.config["run_option"]["out_cat_only"]:
# isUpdated, pos_shear = obj.drawObj_slitless(
# tel=self.tel,
# pos_img=pos_img,
# psf_model=psf_model,
# bandpass_list=filt.bandpass_sub_list,
# filt=filt,
# chip=chip,
# g1=obj.g1,
# g2=obj.g2,
# exptime=pointing.exp_time,
# normFilter=norm_filt,
# fd_shear=fd_shear)
# if isUpdated == 1 and self.config["run_option"]["out_psf"]:
# obj.drawObj_PSF(
# tel=self.tel,
# pos_img=pos_img,
# psf_model=psf_model,
# bandpass_list=filt.bandpass_sub_list,
# filt=filt,
# chip=chip,
# g1=obj.g1,
# g2=obj.g2,
# exptime=pointing.exp_time,
# fd_shear=fd_shear,
# chip_output=chip_output)
# if isUpdated == 1:
# # TODO: add up stats
# chip_output.cat_add_obj(obj, pos_img, pos_shear)
# pass
# elif isUpdated == 0:
# missed_obj += 1
# chip_output.Log_error("Objected missed: %s"%(obj.id))
# else:
# chip_output.Log_error("Draw error, object omitted: %s"%(obj.id))
# continue
# except Exception as e:
# traceback.print_exc()
# chip_output.Log_error(e)
# # # [C6 TEST]
# # chip_output.Log_info("check running:1: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
# # chip_output.Log_info('draw object %s'%obj.id)
# # chip_output.Log_info('mag = %.3f'%obj.param['mag_use_normal'])
# # Unload SED:
# obj.unload_SED()
# del obj
# gc.collect()
# del psf_model
# del self.cat
# gc.collect()
chip_output.Log_error("Failed simulation on step: %s"%(step))
break
chip_output.Log_info("check running:1: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
# Detector Effects
# ===========================================================
# whether to output zero, dark, flat calibration images.
# if not self.config["run_option"]["out_cat_only"]:
# chip.img = chip.addEffects(
# config=self.config,
# img=chip.img,
# chip_output=chip_output,
# filt=filt,
# ra_cen=pointing.ra,
# dec_cen=pointing.dec,
# img_rot=pointing.img_pa,
# exptime=pointing.exp_time,
# pointing_ID=pointing.id,
# timestamp_obs=pointing.timestamp,
# pointing_type=pointing.pointing_type,
# sky_map=sky_map, tel = self.tel,
# logger=chip_output.logger)
# if pointing.pointing_type == 'SCIE':
# datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
# date_obs = datetime_obs.strftime("%y%m%d")
# time_obs = datetime_obs.strftime("%H%M%S")
# h_prim = generatePrimaryHeader(
# xlen=chip.npix_x,
# ylen=chip.npix_y,
# pointNum = str(pointing.id),
# ra=pointing.ra,
# dec=pointing.dec,
# pixel_scale=chip.pix_scale,
# date=date_obs,
# time_obs=time_obs,
# exptime=pointing.exp_time,
# im_type='SCI',
# sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z],
# sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz],
# project_cycle=self.config["project_cycle"],
# run_counter=self.config["run_counter"],
# chip_name=str(chip.chipID).rjust(2, '0'))
# h_ext = generateExtensionHeader(
# chip=chip,
# xlen=chip.npix_x,
# ylen=chip.npix_y,
# ra=pointing.ra,
# dec=pointing.dec,
# pa=pointing.img_pa.deg,
# gain=chip.gain,
# readout=chip.read_noise,
# dark=chip.dark_noise,
# saturation=90000,
# pixel_scale=chip.pix_scale,
# pixel_size=chip.pix_size,
# xcen=chip.x_cen,
# ycen=chip.y_cen,
# extName='SCI',
# timestamp=pointing.timestamp,
# exptime=pointing.exp_time,
# readoutTime=chip.readout_time)
# chip.img = galsim.Image(chip.img.array, dtype=np.uint16)
# hdu1 = fits.PrimaryHDU(header=h_prim)
# hdu1.add_checksum()
# hdu1.header.comments['CHECKSUM'] = 'HDU checksum'
# hdu1.header.comments['DATASUM'] = 'data unit checksum'
# hdu2 = fits.ImageHDU(chip.img.array, header=h_ext)
# hdu2.add_checksum()
# hdu2.header.comments['XTENSION'] = 'extension type'
# hdu2.header.comments['CHECKSUM'] = 'HDU checksum'
# hdu2.header.comments['DATASUM'] = 'data unit checksum'
# hdu1 = fits.HDUList([hdu1, hdu2])
# fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits')
# hdu1.writeto(fname, output_verify='ignore', overwrite=True)
# chip_output.Log_info("# objects that are too bright %d out of %d"%(bright_obj, self.nobj))
# chip_output.Log_info("# objects that are too dim %d out of %d"%(dim_obj, self.nobj))
# chip_output.Log_info("# objects that are missed %d out of %d"%(missed_obj, self.nobj))
del chip.img
chip_output.Log_info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
def run_one_chip_calibration(self, chip, filt, pointing, chip_output, skyback_level = 20000, sky_level_filt = 'g', wcs_fp=None, psf_model=None, cat_dir=None, sed_dir=None):
# # Get WCS for the focal plane
# if wcs_fp == None:
# wcs_fp = self.focal_plane.getTanWCS(ra_cen, dec_cen, pointing.img_pa, chip.pix_scale)
# Create chip Image
chip.img = galsim.ImageF(chip.npix_x, chip.npix_y)
chip.img.setOrigin(chip.bound.xmin, chip.bound.ymin)
# chip.img.wcs = wcs_fp
pf_map = np.zeros_like(chip.img.array)
if self.config["obs_setting"]["LED_TYPE"] is not None:
if len(self.config["obs_setting"]["LED_TYPE"]) != 0:
print("LED OPEN--------")
led_obj = FlatLED(chip, filt)
led_flat = led_obj.drawObj_LEDFlat(led_type_list=self.config["obs_setting"]["LED_TYPE"], exp_t_list=self.config["obs_setting"]["LED_TIME"])
pf_map = led_flat
# whether to output zero, dark, flat calibration images.
expTime = self.config["obs_setting"]["exp_time"]
norm_scaler = skyback_level/expTime / self.filter_param.param[sky_level_filt][5]
if skyback_level == 0:
self.config["ins_effects"]["shutter_effect"] = False
if chip.survey_type == "photometric":
sky_map = np.ones_like(chip.img.array) * norm_scaler * self.filter_param.param[chip.filter_type][5] / self.tel.pupil_area
elif chip.survey_type == "spectroscopic":
flat_normal = np.ones_like(chip.img.array)
if self.config["ins_effects"]["flat_fielding"] == True:
chip_output.Log_info("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding" % chip.chipID)
msg = str(chip.img.bounds)
chip_output.Log_info(msg)
flat_img = Effects.MakeFlatSmooth(
chip.img.bounds,
int(self.config["random_seeds"]["seed_flat"]))
flat_normal = flat_normal * flat_img.array / np.mean(flat_img.array)
if self.config["ins_effects"]["shutter_effect"] == True:
chip_output.Log_info("SLS flat preprocess,CHIP %d : Apply shutter effect" % chip.chipID)
shuttimg = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735,
dt=1E-3) # shutter effect normalized image for this chip
flat_normal = flat_normal * shuttimg
flat_normal = np.array(flat_normal, dtype='float32')
sky_map = calculateSkyMap_split_g(
skyMap=flat_normal,
blueLimit=filt.blue_limit,
redLimit=filt.red_limit,
conf=chip.sls_conf,
pixelSize=chip.pix_scale,
isAlongY=0,
flat_cube=chip.flat_cube)
sky_map = sky_map * norm_scaler
chip.img = chip.addEffects(
config=self.config,
img=chip.img,
chip_output=chip_output,
filt=filt,
ra_cen=pointing.ra,
dec_cen=pointing.dec,
img_rot=pointing.img_pa,
exptime=self.config["obs_setting"]["exp_time"],
pointing_ID=pointing.id,
timestamp_obs=pointing.timestamp,
pointing_type=pointing.pointing_type,
sky_map=sky_map, tel=self.tel,
post_flash_map=pf_map,
logger=chip_output.logger)
datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
date_obs = datetime_obs.strftime("%y%m%d")
time_obs = datetime_obs.strftime("%H%M%S")
h_prim = generatePrimaryHeader(
xlen=chip.npix_x,
ylen=chip.npix_y,
pointNum=str(pointing.id),
ra=pointing.ra,
dec=pointing.dec,
pixel_scale=chip.pix_scale,
date=date_obs,
time_obs=time_obs,
exptime=self.config["obs_setting"]["exp_time"],
im_type='DARKPF',
sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z],
sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz],
chip_name=str(chip.chipID).rjust(2, '0'))
h_ext = generateExtensionHeader(
chip=chip,
xlen=chip.npix_x,
ylen=chip.npix_y,
ra=pointing.ra,
dec=pointing.dec,
pa=pointing.img_pa.deg,
gain=chip.gain,
readout=chip.read_noise,
dark=chip.dark_noise,
saturation=90000,
pixel_scale=chip.pix_scale,
pixel_size=chip.pix_size,
xcen=chip.x_cen,
ycen=chip.y_cen,
extName='SCI',
timestamp=pointing.timestamp,
exptime=self.config["obs_setting"]["exp_time"],
readoutTime=chip.readout_time)
chip.img = galsim.Image(chip.img.array, dtype=np.uint16)
hdu1 = fits.PrimaryHDU(header=h_prim)
hdu1.add_checksum()
hdu1.header.comments['CHECKSUM'] = 'HDU checksum'
hdu1.header.comments['DATASUM'] = 'data unit checksum'
hdu2 = fits.ImageHDU(chip.img.array, header=h_ext)
hdu2.add_checksum()
hdu2.header.comments['XTENSION'] = 'extension type'
hdu2.header.comments['CHECKSUM'] = 'HDU checksum'
hdu2.header.comments['DATASUM'] = 'data unit checksum'
hdu1 = fits.HDUList([hdu1, hdu2])
fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits')
hdu1.writeto(fname, output_verify='ignore', overwrite=True)
# chip_output.Log_info("# objects that are too bright %d out of %d" % (bright_obj, self.nobj))
# chip_output.Log_info("# objects that are too dim %d out of %d" % (dim_obj, self.nobj))
# chip_output.Log_info("# objects that are missed %d out of %d" % (missed_obj, self.nobj))
del chip.img
chip_output.Log_info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB" % (
pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024)))
# def run_one_chip_calibration(self, chip, filt, pointing, chip_output, skyback_level = 20000, sky_level_filt = 'g', wcs_fp=None, psf_model=None, cat_dir=None, sed_dir=None):
# # # Get WCS for the focal plane
# # if wcs_fp == None:
# # wcs_fp = self.focal_plane.getTanWCS(ra_cen, dec_cen, pointing.img_pa, chip.pix_scale)
# # Create chip Image
# chip.img = galsim.ImageF(chip.npix_x, chip.npix_y)
# chip.img.setOrigin(chip.bound.xmin, chip.bound.ymin)
# # chip.img.wcs = wcs_fp
# pf_map = np.zeros_like(chip.img.array)
# if self.config["obs_setting"]["LED_TYPE"] is not None:
# if len(self.config["obs_setting"]["LED_TYPE"]) != 0:
# print("LED OPEN--------")
# led_obj = FlatLED(chip, filt)
# led_flat = led_obj.drawObj_LEDFlat(led_type_list=self.config["obs_setting"]["LED_TYPE"], exp_t_list=self.config["obs_setting"]["LED_TIME"])
# pf_map = led_flat
# # whether to output zero, dark, flat calibration images.
# expTime = self.config["obs_setting"]["exp_time"]
# norm_scaler = skyback_level/expTime / self.filter_param.param[sky_level_filt][5]
# if skyback_level == 0:
# self.config["ins_effects"]["shutter_effect"] = False
# if chip.survey_type == "photometric":
# sky_map = np.ones_like(chip.img.array) * norm_scaler * self.filter_param.param[chip.filter_type][5] / self.tel.pupil_area
# elif chip.survey_type == "spectroscopic":
# flat_normal = np.ones_like(chip.img.array)
# if self.config["ins_effects"]["flat_fielding"] == True:
# chip_output.Log_info("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding" % chip.chipID)
# msg = str(chip.img.bounds)
# chip_output.Log_info(msg)
# flat_img = Effects.MakeFlatSmooth(
# chip.img.bounds,
# int(self.config["random_seeds"]["seed_flat"]))
# flat_normal = flat_normal * flat_img.array / np.mean(flat_img.array)
# if self.config["ins_effects"]["shutter_effect"] == True:
# chip_output.Log_info("SLS flat preprocess,CHIP %d : Apply shutter effect" % chip.chipID)
# shuttimg = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735,
# dt=1E-3) # shutter effect normalized image for this chip
# flat_normal = flat_normal * shuttimg
# flat_normal = np.array(flat_normal, dtype='float32')
# sky_map = calculateSkyMap_split_g(
# skyMap=flat_normal,
# blueLimit=filt.blue_limit,
# redLimit=filt.red_limit,
# conf=chip.sls_conf,
# pixelSize=chip.pix_scale,
# isAlongY=0,
# flat_cube=chip.flat_cube)
# sky_map = sky_map * norm_scaler
# chip.img = chip.addEffects(
# config=self.config,
# img=chip.img,
# chip_output=chip_output,
# filt=filt,
# ra_cen=pointing.ra,
# dec_cen=pointing.dec,
# img_rot=pointing.img_pa,
# exptime=self.config["obs_setting"]["exp_time"],
# pointing_ID=pointing.id,
# timestamp_obs=pointing.timestamp,
# pointing_type=pointing.pointing_type,
# sky_map=sky_map, tel=self.tel,
# post_flash_map=pf_map,
# logger=chip_output.logger)
# datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
# date_obs = datetime_obs.strftime("%y%m%d")
# time_obs = datetime_obs.strftime("%H%M%S")
# h_prim = generatePrimaryHeader(
# xlen=chip.npix_x,
# ylen=chip.npix_y,
# pointNum=str(pointing.id),
# ra=pointing.ra,
# dec=pointing.dec,
# pixel_scale=chip.pix_scale,
# date=date_obs,
# time_obs=time_obs,
# exptime=self.config["obs_setting"]["exp_time"],
# im_type='DARKPF',
# sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z],
# sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz],
# chip_name=str(chip.chipID).rjust(2, '0'))
# h_ext = generateExtensionHeader(
# chip=chip,
# xlen=chip.npix_x,
# ylen=chip.npix_y,
# ra=pointing.ra,
# dec=pointing.dec,
# pa=pointing.img_pa.deg,
# gain=chip.gain,
# readout=chip.read_noise,
# dark=chip.dark_noise,
# saturation=90000,
# pixel_scale=chip.pix_scale,
# pixel_size=chip.pix_size,
# xcen=chip.x_cen,
# ycen=chip.y_cen,
# extName='SCI',
# timestamp=pointing.timestamp,
# exptime=self.config["obs_setting"]["exp_time"],
# readoutTime=chip.readout_time)
# chip.img = galsim.Image(chip.img.array, dtype=np.uint16)
# hdu1 = fits.PrimaryHDU(header=h_prim)
# hdu1.add_checksum()
# hdu1.header.comments['CHECKSUM'] = 'HDU checksum'
# hdu1.header.comments['DATASUM'] = 'data unit checksum'
# hdu2 = fits.ImageHDU(chip.img.array, header=h_ext)
# hdu2.add_checksum()
# hdu2.header.comments['XTENSION'] = 'extension type'
# hdu2.header.comments['CHECKSUM'] = 'HDU checksum'
# hdu2.header.comments['DATASUM'] = 'data unit checksum'
# hdu1 = fits.HDUList([hdu1, hdu2])
# fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits')
# hdu1.writeto(fname, output_verify='ignore', overwrite=True)
# # chip_output.Log_info("# objects that are too bright %d out of %d" % (bright_obj, self.nobj))
# # chip_output.Log_info("# objects that are too dim %d out of %d" % (dim_obj, self.nobj))
# # chip_output.Log_info("# objects that are missed %d out of %d" % (missed_obj, self.nobj))
# del chip.img
# chip_output.Log_info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB" % (
# pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024)))
def runExposure_MPI_PointingList(self, pointing_list, chips=None, use_mpi=False):
if use_mpi:
......@@ -587,11 +263,11 @@ class Observation(object):
pointing_ID = pointing.id
self.focal_plane = FocalPlane(chip_list=pointing.obs_param["run_chips"])
# Make Chip & Filter lists
self.chip_list = []
self.filter_list = []
self.all_filters = []
for i in range(self.focal_plane.nchips):
chipID = i + 1
self.chip_list = []
self.filter_list = []
self.all_filters = []
chip = Chip(chipID=chipID, config=self.config)
filter_id, filter_type = chip.getChipFilter()
filt = Filter(
......@@ -641,7 +317,9 @@ class Observation(object):
pointing_type=pointing.pointing_type,
pointing_ID=pointing_ID,
subdir=sub_img_dir,
prefix=prefix)
prefix=prefix,
ra_cen=pointing.ra,
dec_cen=pointing.dec)
chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid))
self.run_one_chip(
chip=chip,
......
ObservationSim/SimSteps.py
View file @ 8f6e5f8f
......@@ -98,7 +98,7 @@ class SimSteps:
zoldial_spec = filt.zodical_spec)
sky_map = sky_map + filt.sky_background
sky_map = sky_map * tel.pupil_area * obs_param["exptime"]
# sky_map = sky_map * tel.pupil_area * obs_param["exptime"]
chip.img += sky_map
return chip, filt, tel, pointing
......@@ -292,11 +292,11 @@ class SimSteps:
return chip, filt, tel, pointing
def add_cosmic_rays(self, chip, filt, tel, pointing, catalog, obs_param):
self.chip_output.Log_info(msg=" Adding Cosmic-Ray", logger=self.logger)
self.chip_output.Log_info(" Adding Cosmic-Ray")
chip.img, crmap_gsimg, cr_event_num = chip_utils.add_cosmic_rays(
img=chip.img,
chip=chip,
exptime=pointing.exptime,
exptime=pointing.exp_time,
seed=self.overall_config["random_seeds"]["seed_CR"]+pointing.id*30+chip.chipID)
# [TODO] output cosmic ray image
return chip, filt, tel, pointing
......@@ -310,11 +310,11 @@ class SimSteps:
InputDark = False
if obs_param["add_dark"] == True:
if InputDark:
chip.img = chip_utils.add_inputdark(img=chip.img, chip=chip, exptime=pointing.exptime)
chip.img = chip_utils.add_inputdark(img=chip.img, chip=chip, exptime=pointing.exp_time)
else:
chip.img, _ = chip_utils.add_poisson(img=chip.img, chip=chip, exptime=pointing.exptime, poisson_noise=chip.poisson_noise)
chip.img, _ = chip_utils.add_poisson(img=chip.img, chip=chip, exptime=pointing.exp_time, poisson_noise=chip.poisson_noise)
else:
chip.img, _ = chip_utils.add_poisson(img=chip.img, chip=self, exptime=pointing.exptime, poisson_noise=chip.poisson_noise, dark_noise=0.)
chip.img, _ = chip_utils.add_poisson(img=chip.img, chip=self, exptime=pointing.exp_time, poisson_noise=chip.poisson_noise, dark_noise=0.)
return chip, filt, tel, pointing
def add_brighter_fatter(self, chip, filt, tel, pointing, catalog, obs_param):
......@@ -360,7 +360,7 @@ class SimSteps:
newimg = galsim.Image(nx, int(ny+chip.overscan_y), init_value=0)
for ichannel in range(16):
print('\n***add CTI effects: pointing-{:} chip-{:} channel-{:}***'.format(pointing.id, chip.chipID, ichannel+1))
noverscan, nsp, nmax = self.overscan_y, 3, 10
noverscan, nsp, nmax = chip.overscan_y, 3, 10
beta, w, c = 0.478, 84700, 0
t = np.array([0.74, 7.7, 37],dtype=np.float32)
rho_trap = np.array([0.6, 1.6, 1.4],dtype=np.float32)
......@@ -412,7 +412,7 @@ class SimSteps:
if obs_param["gain_16channel"] == True:
chip.img, chip.gain_channel = Effects.ApplyGainNonUniform16(
chip.img, gain=chip.gain,
nsecy = self.nsecy, nsecx=self.nsecx,
nsecy = chip.nsecy, nsecx=chip.nsecx,
seed=self.overall_config["random_seeds"]["seed_gainNonUniform"]+chip.chipID)
elif obs_param["gain_16channel"] == False:
chip.img /= chip.gain
......
ObservationSim/_util.py
View file @ 8f6e5f8f
......@@ -29,25 +29,9 @@ def generate_pointing_list(config, pointing_filename=None, data_dir=None):
# Calculate starting time(s) for CAL exposures
# NOTE: temporary implementation
t = datetime.timestamp(t0)
ncal = config['obs_setting']['np_cal']
ipoint = 0
for i in range(ncal):
pointing = Pointing(
id = ipoint,
ra=config["obs_setting"]["ra_center"],
dec=config["obs_setting"]["dec_center"],
img_pa=config["obs_setting"]["image_rot"],
timestamp=t,
pointing_type='CAL')
t += 3 * delta_t * 60. # 3 calibration exposures for each pointing
pointing_list.append(pointing)
ipoint += 1
run_pointings = config['obs_setting']['run_pointings']
if config["obs_setting"]["exp_time"]:
exp_time = config["obs_setting"]["exp_time"]
else:
exp_time = 150.
if pointing_filename and data_dir:
pointing_file = os.path.join(data_dir, pointing_filename)
......@@ -62,14 +46,18 @@ def generate_pointing_list(config, pointing_filename=None, data_dir=None):
continue
line = line.strip()
columns = line.split()
pointing = Pointing(exp_time=exp_time)
pointing.read_pointing_columns(columns=columns, id=ipoint, t=t)
pointing = Pointing()
pointing.read_pointing_columns(columns=columns, id=ipoint)
t += delta_t * 60.
pointing_list.append(pointing)
iline += 1
ipoint += 1
f.close()
else:
if config["obs_setting"]["exp_time"]:
exp_time = config["obs_setting"]["exp_time"]
else:
exp_time = 150.
pointing = Pointing(
id=ipoint,
ra=config["obs_setting"]["ra_center"],
......@@ -96,38 +84,8 @@ def make_run_dirs(work_dir, run_name, pointing_list):
os.makedirs(imgDir, exist_ok=True)
except OSError:
pass
# prefix = "MSC_"
# for pointing in pointing_list:
# fname=prefix + str(pointing.id).rjust(7, '0')
# subImgDir = os.path.join(imgDir, fname)
# if not os.path.exists(subImgDir):
# try:
# os.makedirs(subImgDir, exist_ok=True)
# except OSError:
# pass
return imgDir
# def imgName(tt=0):
# ut = datetime.utcnow()
# eye, emo, eda, eho, emi, ese = str(ut.year), str(ut.month), str(ut.day), str(ut.hour), str(ut.minute), str(ut.second)
# emse = str(ut.microsecond)
# if int(emo)<10: emo = "0%s"%emo
# if int(eda)<10: eda = "0%s"%eda
# if int(eho)<10: eho = "0%s"%eho
# if int(emi)<10: emi = "0%s"%emi
# if int(ese)<10: ese = "0%s"%ese
# if tt==0:
# namekey = "CSST%s%s%sT%s%s%s"%(eye,emo,eda,eho,emi,ese)
# elif tt==1:
# namekey = "%s-%s-%sT%s:%s:%s.%s"%(eye,emo,eda,eho,emi,ese,emse)
# elif tt==2:
# namekey = "%s%s%s%s%s%s"%(eye,emo,eda,eho,emi,ese)
# else:
# raise ValueError("!!! Give a right 'tt' value.")
# return namekey
def makeSubDir_PointingList(path_dict, config, pointing_ID=0):
imgDir = os.path.join(path_dict["work_dir"], config["run_name"])
if not os.path.exists(imgDir):
......
config/config_C6_dev_calib.yaml deleted 100644 → 0
View file @ 633fbbab
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/Users/zhangxin/Work/SlitlessSim/CSST_SIM/CSST_new_sim/csst-simulation/"
data_dir: "/Volumes/EAGET/C6_data/inputData/"
run_name: "C6_new_sim_2sq_run2"
project_cycle: 6
run_counter: 1
# Whether to use MPI
run_option:
use_mpi: NO
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_C6_20221212"
star_cat:
galaxy_cat:
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
CALIB_cat: "calibration_data/GP/calibrationCat_CHIP1_GI.fits"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Catalog_C6_20221212/sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
CALIB_SED: "calibration_data/GP/GPTable/"
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
# "CALIBRATION": falt, bias, dark with or without postflash
survey_type: "Spectroscopic"
#"LED": ['LED1','LED2','LED3','LED4','LED5','LED6','LED7','LED8','LED9','LED10','LED11','LED12','LED13','LED14'] or null
#'LED1': '275', 'LED2': '310', 'LED3': '430', 'LED4': '505', 'LED5': '545', 'LED6': '590', 'LED7': '670',
#'LED8': '760', 'LED9': '940', 'LED10': '940', 'LED11': '1050', 'LED12': '1050','LED13': '340', 'LED14': '365'
LED_TYPE: ['LED5','LED3']
LED_TIME: [1.,0.1]
# unit e- ,flat level
FLAT_LEVEL: 20000
FLAT_LEVEL_FIL: 'g'
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 192.8595
dec_center: 27.1283
# Image rotation [degree]
image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/Volumes/EAGET/C6_data/inputData/"
pointing_file: "pointing_radec_246.5_40.dat"
# Number of calibration pointings
np_cal: 0
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [1]
# Whether to enable astrometric modeling
enable_astrometric_model: False
# Whether to enable straylight model
enable_straylight_model: False
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
# mag_sat_margin: -2.5
mag_sat_margin: -15.
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Gauss"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
psf_sls_dir: "/share/simudata/CSSOSDataProductsSims/data/SLS_PSF_PCA_fp/"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog
shear_type: "catalog"
# For constant shear filed
reduced_g1: 0.
reduced_g2: 0.
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: NO # Whether to add field distortions
add_back: YES # Whether to add sky background
add_dark: YES # Whether to add dark noise
add_readout: YES # Whether to add read-out (Gaussian) noise
add_bias: YES # Whether to add bias-level to images
add_prescan: OFF
bias_16channel: YES # Whether to add different biases for 16 channels
gain_16channel: YES # Whether to make different gains for 16 channels
shutter_effect: YES # Whether to add shutter effect
flat_fielding: YES # Whether to add flat-fielding effect
prnu_effect: YES # Whether to add PRNU effect
non_linear: YES # Whether to add non-linearity
cosmic_ray: NO # Whether to add cosmic-ray
cray_differ: NO # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: YES # Whether to simulate CTE trails
saturbloom: YES # Whether to simulate Saturation & Blooming
add_badcolumns: YES # Whether to add bad columns
add_hotpixels: YES # Whether to add hot pixels
add_deadpixels: YES # Whether to add dead(dark) pixels
bright_fatter: YES # Whether to simulate Brighter-Fatter (also diffusion) effect
format_output: OFF
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 40 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 500 # bias level [e-/pixel]
# gain: 1.1 # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
config/config_C6_test_wcs.yaml deleted 100644 → 0
View file @ 633fbbab
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/share/home/fangyuedong/csst-simulation/workplace/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "fd_shear_g2m"
# Whether to use MPI
run_option:
use_mpi: YES
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_C6_20221212"
star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
galaxy_cat: "cat2CSSTSim_bundle/"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Catalog_C6_20221212/sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: YES
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "All"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 192.8595
dec_center: 27.1283
# Image rotation [degree]
image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_radec_246.5_40.dat"
# Number of calibration pointings
np_cal: 0
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [9]
# Whether to enable astrometric modeling
enable_astrometric_model: False
# Whether to enable straylight model
enable_straylight_model: False
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog
shear_type: "catalog"
# For constant shear filed
reduced_g1: 0.
reduced_g2: 0.
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: YES # Whether to add field distortions
add_back: YES # Whether to add sky background
add_dark: YES # Whether to add dark noise
add_readout: YES # Whether to add read-out (Gaussian) noise
add_bias: YES # Whether to add bias-level to images
bias_16channel: OFF # Whether to add different biases for 16 channels
gain_16channel: OFF # Whether to make different gains for 16 channels
shutter_effect: OFF # Whether to add shutter effect
flat_fielding: OFF # Whether to add flat-fielding effect
prnu_effect: OFF # Whether to add PRNU effect
non_linear: OFF # Whether to add non-linearity
cosmic_ray: OFF # Whether to add cosmic-ray
cray_differ: OFF # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: OFF # Whether to simulate CTE trails
saturbloom: OFF # Whether to simulate Saturation & Blooming
add_badcolumns: OFF # Whether to add bad columns
add_hotpixels: OFF # Whether to add hot pixels
add_deadpixels: OFF # Whether to add dead(dark) pixels
bright_fatter: OFF # Whether to simulate Brighter-Fatter (also diffusion) effect
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 40 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 500 # bias level [e-/pixel]
# gain: 1.1 # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
config/config_NGP.yaml deleted 100644 → 0
View file @ 633fbbab
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/share/home/fangyuedong/test_psf_rot/csst-simulation/workplace/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "NGP_C3"
# Whether to use MPI
run_option:
use_mpi: YES
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "OnOrbitCalibration/CTargets20211231"
star_cat: "CT-NGP_r1.8_G28.hdf5"
galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Templates/Galaxy/"
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 192.8595
dec_center: 27.1283
# Image rotation [degree]
image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_test_NGP_2.17.dat"
# Number of calibration pointings
np_cal: 0
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0, 1, 2]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: null
# Whether to enable astrometric modeling
enable_astrometric_model: True
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog
shear_type: "constant"
# For constant shear filed
reduced_g1: 0.026
reduced_g2: 0.015
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise
add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images
bias_16channel: ON # Whether to add different biases for 16 channels
gain_16channel: ON # Whether to make different gains for 16 channels
shutter_effect: ON # Whether to add shutter effect
flat_fielding: ON # Whether to add flat-fielding effect
prnu_effect: ON # Whether to add PRNU effect
non_linear: ON # Whether to add non-linearity
cosmic_ray: ON # Whether to add cosmic-ray
cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: ON # Whether to simulate CTE trails
saturbloom: ON # Whether to simulate Saturation & Blooming
add_badcolumns: ON # Whether to add bad columns
add_hotpixels: ON # Whether to add hot pixels
add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 40 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 500 # bias level [e-/pixel]
# gain: 1.1 # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
config/config_example.yaml deleted 100644 → 0
View file @ 633fbbab
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/public/home/fangyuedong/20211203/CSST/workplace/"
data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
run_name: "example"
# Whether to use MPI
run_option:
use_mpi: YES
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "catalog_points_7degree2/point_RA60.9624_DE-41.5032/"
star_cat: "stars_ccd13_p_RA60.9624_DE-41.5032.hdf5"
galaxy_cat: null
SED_templates_path:
star_SED: "SED_MMW_Gaia_Cluster_D20_SS.hdf5"
galaxy_SED: null
# Only simulate stars?
star_only: YES
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 60.9624
dec_center: -41.5032
# Image rotation [degree]
image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: null
pointing_file: null
# Number of calibration pointings
np_cal: 2
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: null
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [18]
# Whether to enable astrometric modeling
enable_astrometric_model: True
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog
shear_type: "constant"
# For constant shear filed
reduced_g1: 0.026
reduced_g2: 0.015
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise
add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images
bias_16channel: ON # Whether to add different biases for 16 channels
gain_16channel: ON # Whether to make different gains for 16 channels
shutter_effect: ON # Whether to add shutter effect
flat_fielding: ON # Whether to add flat-fielding effect
prnu_effect: ON # Whether to add PRNU effect
non_linear: OFF # Whether to add non-linearity
cosmic_ray: ON # Whether to add cosmic-ray
cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: ON # Whether to simulate CTE trails
saturbloom: ON # Whether to simulate Saturation & Blooming
add_badcolumns: ON # Whether to add bad columns
add_hotpixels: ON # Whether to add hot pixels
add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 40 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 500 # bias level [e-/pixel]
# gain: 1.1 # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
config/config_fgs.yaml deleted 100644 → 0
View file @ 633fbbab
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/share/home/fangyuedong/csst-simulation/workplace/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "fgs_test"
# Whether to use MPI
run_option:
use_mpi: NO
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_C6_20221212"
star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
galaxy_cat: "cat2CSSTSim_bundle/"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Catalog_C6_20221212/sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "FGS"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 244.972773
dec_center: 39.895901
# Image rotation [degree]
image_rot: 109.59
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
# pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
# pointing_file: "pointing_radec_246.5_40.dat"
pointing_dir: null
pointing_file: null
# Number of calibration pointings
np_cal: 0
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [32]
# Whether to enable astrometric modeling
enable_astrometric_model: True
# Whether to enable straylight model
enable_straylight_model: True
# Cut by saturation magnitude in which band?
cut_in_band: "FGS"
# saturation magnitude margin
mag_sat_margin: -2.5
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog
shear_type: "catalog"
# For constant shear filed
reduced_g1: 0.
reduced_g2: 0.
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise
add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images
bias_16channel: OFF # Whether to add different biases for 16 channels
gain_16channel: OFF # Whether to make different gains for 16 channels
shutter_effect: OFF # Whether to add shutter effect
flat_fielding: ON # Whether to add flat-fielding effect
prnu_effect: ON # Whether to add PRNU effect
non_linear: ON # Whether to add non-linearity
cosmic_ray: ON # Whether to add cosmic-ray
cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: ON # Whether to simulate CTE trails
saturbloom: ON # Whether to simulate Saturation & Blooming
add_badcolumns: ON # Whether to add bad columns
add_hotpixels: ON # Whether to add hot pixels
add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 0.01 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 2000 # bias level [e-/pixel]
# gain: 1. # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
config/config_overall.yaml
View file @ 8f6e5f8f
......@@ -12,7 +12,7 @@
# ok to pass either way or both, as long as they are consistent
work_dir: "/share/home/fangyuedong/20231211/workplace/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "test20231218_c6_onlyCat"
run_name: "test_new_config"
# Project cycle and run counter are used to name the outputs
project_cycle: 6
......@@ -28,7 +28,7 @@ run_option:
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: YES
out_cat_only: NO
###############################################
# Catalog setting
......@@ -49,7 +49,7 @@ catalog_options:
AGN_SED_WAVE: "wave_ross13.npy"
# Only simulate stars?
star_only: YES
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
......@@ -70,6 +70,12 @@ obs_setting:
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_radec_246.5_40.dat"
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
# Whether to enable astrometric modeling
enable_astrometric_model: True
......
config/obs_config_SCI_WIDE_phot.yaml
View file @ 8f6e5f8f
......@@ -27,7 +27,7 @@ call_sequence:
exptime: 150. # [s]
shutter_effect: YES
flat_fielding: YES
enable_straylight_model: True
enable_straylight_model: YES
# Apply PRNU to accumulated photons
PRNU_effect: {}
# Accumulate photons caused by cosmic rays
......@@ -46,7 +46,7 @@ call_sequence:
nonlinearity: {}
# Apply CCD Saturation & Blooming
blooming: {}
# Run CTE simulation
# # Run CTE simulation
CTE_effect: {}
# Add prescan and overscan
prescan_overscan: {}
......@@ -58,4 +58,6 @@ call_sequence:
# Apply gain
gain:
gain_16channel: YES
# Output the final image
quantization_and_output: {}
...
\ No newline at end of file
config/test_fd_C6.yaml deleted 100644 → 0
View file @ 633fbbab
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/share/home/fangyuedong/csst-simulation/workplace/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "fd_shear_g2m"
# Whether to use MPI
run_option:
use_mpi: NO
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_C6_20221212"
star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
galaxy_cat: "cat2CSSTSim_bundle/"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Catalog_C6_20221212/sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: YES
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 192.8595
dec_center: 27.1283
# Image rotation [degree]
image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_radec_246.5_40.dat"
# Number of calibration pointings
np_cal: 0
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [9]
# Whether to enable astrometric modeling
enable_astrometric_model: True
# Whether to enable straylight model
enable_straylight_model: False
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog
shear_type: "catalog"
# For constant shear filed
reduced_g1: 0.
reduced_g2: 0.
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: YES # Whether to add field distortions
add_back: YES # Whether to add sky background
add_dark: YES # Whether to add dark noise
add_readout: NO # Whether to add read-out (Gaussian) noise
add_bias: YES # Whether to add bias-level to images
bias_16channel: NO # Whether to add different biases for 16 channels
gain_16channel: NO # Whether to make different gains for 16 channels
shutter_effect: NO # Whether to add shutter effect
flat_fielding: YES # Whether to add flat-fielding effect
prnu_effect: NO # Whether to add PRNU effect
non_linear: NO # Whether to add non-linearity
cosmic_ray: NO # Whether to add cosmic-ray
cray_differ: NO # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: NO # Whether to simulate CTE trails
saturbloom: NO # Whether to simulate Saturation & Blooming
add_badcolumns: NO # Whether to add bad columns
add_hotpixels: NO # Whether to add hot pixels
add_deadpixels: NO # Whether to add dead(dark) pixels
bright_fatter: NO # Whether to simulate Brighter-Fatter (also diffusion) effect
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 40 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 500 # bias level [e-/pixel]
# gain: 1.1 # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
run_NGP.pbs deleted 100755 → 0
View file @ 633fbbab
#!/bin/bash
#PBS -N SIMS
#PBS -l nodes=wcl-1:ppn=16+wcl-2:ppn=16+wcl-3:ppn=16+wcl-4:ppn=16+wcl-5:ppn=16+wcl-6:ppn=16
#PBS -u fangyuedong
###PBS -j oe
cd $PBS_O_WORKDIR
NP=96
date
mpirun -np $NP python3 /share/home/fangyuedong/fgs_sim/csst-simulation/run_sim.py \
--config_file config_NGP.yaml \
--catalog NGPCatalog \
-c /share/home/fangyuedong/fgs_sim/csst-simulation/config
run_fgs.pbs deleted 100755 → 0
View file @ 633fbbab
#!/bin/bash
#PBS -N SIMS
#PBS -l nodes=wcl-1:ppn=12
###PBS -l nodes=wcl-1:ppn=24+wcl-2:ppn=24+wcl-3:ppn=24+wcl-4:ppn=24+wcl-5:ppn=24+wcl-6:ppn=24
#PBS -u fangyuedong
###PBS -j oe
cd $PBS_O_WORKDIR
NP=40
date
mpirun -np $NP python3 /share/home/fangyuedong/fgs_sim/csst-simulation/run_sim.py \
--config_file config_fgs.yaml \
--catalog FGS_Catalog \
-c /share/home/fangyuedong/fgs_sim/csst-simulation/config
run_sim.py
View file @ 8f6e5f8f
......@@ -50,10 +50,6 @@ def run_sim():
config['work_dir'] = args.work_dir
# Some default values
# if "bias_16channel" not in config["ins_effects"]:
# config["ins_effects"]["bias_16channel"] = False
# if "gain_16channel" not in config["ins_effects"]:
# config["ins_effects"]["gain_16channel"] = False
if "mag_sat_margin" not in config["obs_setting"]:
config["obs_setting"]["mag_sat_margin"] = -2.5
if "mag_lim_margin" not in config["obs_setting"]:
......@@ -93,8 +89,7 @@ def run_sim():
# Run simulation
obs.runExposure_MPI_PointingList(
pointing_list=pointing_list,
use_mpi=config["run_option"]["use_mpi"],
chips=config["obs_setting"]["run_chips"])
use_mpi=config["run_option"]["use_mpi"])
if __name__=='__main__':
run_sim()
test_C6.sh
View file @ 8f6e5f8f
......@@ -3,10 +3,15 @@
date
python3 /share/home/fangyuedong/20231211/csst-simulation/run_sim.py \
--config_file config_C6_dev.yaml \
--config_file config_overall.yaml \
--catalog C6_Catalog \
-c /share/home/fangyuedong/20231211/csst-simulation/config
# python3 /share/home/fangyuedong/20231211/csst-simulation/run_sim.py \
# --config_file config_C6_dev.yaml \
# --catalog C6_Catalog \
# -c /share/home/fangyuedong/20231211/csst-simulation/config
# python3 /share/home/fangyuedong/20231211/csst-simulation/run_sim.py \
# --config_file config_ooc_c6.yaml \
# --catalog C6_Catalog \
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment