Skip to content
GitLab
Commit fd6c3108 authored by Fang Yuedong's avatar Fang Yuedong Committed by Zhang Xin
Browse files

Release v2.0

parent ca7fb5db
release_v3.0 11-csst 29-darktime-exptime-shtopen-shtstat-t current_stable_for_tests develop master milky_way_extinction new_sim release_v2.0 revert-914004c1 sim_scheduler sls_psf_pca_convolve wcs_test_gridGalaxy v3.1.0 v3.0.0 v2.1.0 v2.0.0
2 merge requests!13merge develop into master (for release v2.0),!12Release v2.0
Hide whitespace changes
Inline Side-by-side
Showing
with 1103 additions and 478 deletions
+1103 -478
ObservationSim/MockObject/Quasar.py
+ 5
- 0
View file @ fd6c3108
...@@ -13,6 +13,11 @@ class Quasar(MockObject): ...@@ -13,6 +13,11 @@ class Quasar(MockObject):
super().__init__(param, logger=logger) super().__init__(param, logger=logger)
def load_SED(self, survey_type, sed_path=None, cosids=None, objtypes=None, sed_templates=None, normFilter=None, target_filt=None): def load_SED(self, survey_type, sed_path=None, cosids=None, objtypes=None, sed_templates=None, normFilter=None, target_filt=None):
'''
--------------------------------------------------------------------
(Deprecated) Left over codes from cycle-1 to cycle-3 implementations
--------------------------------------------------------------------
'''
if survey_type == "photometric": if survey_type == "photometric":
norm_thr_rang_ids = normFilter['SENSITIVITY'] > 0.001 norm_thr_rang_ids = normFilter['SENSITIVITY'] > 0.001
if sed_templates is None: if sed_templates is None:
......
ObservationSim/MockObject/SpecDisperser/SpecDisperser.py
+ 3
- 3
View file @ fd6c3108
...@@ -185,8 +185,8 @@ class SpecDisperser(object): ...@@ -185,8 +185,8 @@ class SpecDisperser(object):
origin_in[1] = self.origin[1] origin_in[1] = self.origin[1]
dx0_in = dx[0] dx0_in = dx[0]
dy0_in = dyc[0] dy0_in = dyc[0]
originOut_x = origin_in[1] + dx0_in - 1 originOut_x = origin_in[1] + dx0_in
originOut_y = origin_in[0] + dy0_in - 1 originOut_y = origin_in[0] + dy0_in
if self.flat_cube is None: if self.flat_cube is None:
beam_flat = None beam_flat = None
...@@ -248,7 +248,7 @@ class SpecDisperser(object): ...@@ -248,7 +248,7 @@ class SpecDisperser(object):
if self.isAlongY == 1: if self.isAlongY == 1:
model, _, _ = rotate90(array_orig=model, isClockwise=0) model, _, _ = rotate90(array_orig=model, isClockwise=0)
return model, originOut_x, originOut_y return model, originOut_x, originOut_y, dxpix, dypix, lam_beam,ysens
def writerSensitivityFile(self, conffile='', beam='', w=None, sens=None): def writerSensitivityFile(self, conffile='', beam='', w=None, sens=None):
orders = {'A': '1st', 'B': '0st', 'C': '2st', 'D': '-1st', 'E': '-2st'} orders = {'A': '1st', 'B': '0st', 'C': '2st', 'D': '-1st', 'E': '-2st'}
......
ObservationSim/MockObject/Stamp.py 0 → 100644
+ 126
- 0
View file @ fd6c3108
import os, sys
import random
import numpy as np
import astropy.constants as cons
from astropy.table import Table
from scipy import interpolate
import astropy.io.fits as fitsio
import galsim
import gc
from ObservationSim.MockObject.MockObject import MockObject
from ObservationSim.MockObject._util import magToFlux,VC_A
from ObservationSim.MockObject._util import eObs, integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG,convolveGaussXorders
class Stamp(MockObject):
def __init__(self, param):
super().__init__(param)
def unload_SED(self):
"""(Test) free up SED memory
"""
del self.sed
def drawObj_multiband(self, tel, pos_img, psf_model, bandpass_list, filt, chip, nphotons_tot=None, g1=0, g2=0, exptime=150., fd_shear=None):
if nphotons_tot == None:
nphotons_tot = self.getElectronFluxFilt(filt, tel, exptime)
try:
full = integrate_sed_bandpass(sed=self.sed, bandpass=filt.bandpass_full)
except Exception as e:
print(e)
self.logger.error(e)
return False
nphotons_sum = 0
photons_list = []
xmax, ymax = 0, 0
if self.getMagFilter(filt) <= 15:
folding_threshold = 5.e-4
else:
folding_threshold = 5.e-3
gsp = galsim.GSParams(folding_threshold=folding_threshold)
self.real_pos = self.getRealPos(chip.img, global_x=self.posImg.x, global_y=self.posImg.y,
img_real_wcs=self.real_wcs)
x, y = self.real_pos.x + 0.5, self.real_pos.y + 0.5
x_nominal = int(np.floor(x + 0.5))
y_nominal = int(np.floor(y + 0.5))
dx = x - x_nominal
dy = y - y_nominal
offset = galsim.PositionD(dx, dy)
real_wcs_local = self.real_wcs.local(self.real_pos)
for i in range(len(bandpass_list)):
bandpass = bandpass_list[i]
try:
sub = integrate_sed_bandpass(sed=self.sed, bandpass=bandpass)
except Exception as e:
print(e)
self.logger.error(e)
# return False
continue
ratio = sub/full
if not (ratio == -1 or (ratio != ratio)):
nphotons = ratio * nphotons_tot
else:
# return False
continue
nphotons_sum += nphotons
psf, pos_shear = psf_model.get_PSF(chip=chip, pos_img=pos_img, bandpass=bandpass, folding_threshold=folding_threshold)
_gal = self.param['image']
galIm = galsim.ImageF(_gal, scale=self.param['pixScale'])
gal = galsim.InterpolatedImage(galIm)
gal = gal.withFlux(nphotons)
#gal_shear = galsim.Shear(g1=g1, g2=g2)
#gal = gal.shear(gal_shear)
gal = galsim.Convolve(psf, gal)
if fd_shear is not None:
gal = gal.shear(fd_shear)
stamp = gal.drawImage(wcs=real_wcs_local, method='phot', offset=self.offset, save_photons=True)
xmax = max(xmax, stamp.xmax - stamp.xmin)
ymax = max(ymax, stamp.ymax - stamp.ymin)
photons = stamp.photons
photons.x += x_nominal
photons.y += y_nominal
photons_list.append(photons)
del gal
# print('xmax = %d, ymax = %d '%(xmax, ymax))
stamp = galsim.ImageF(int(xmax*1.1), int(ymax*1.1))
stamp.wcs = real_wcs_local
stamp.setCenter(x_nominal, y_nominal)
bounds = stamp.bounds & galsim.BoundsI(0, chip.npix_x - 1, 0, chip.npix_y - 1)
if bounds.area() > 0:
chip.img.setOrigin(0, 0)
stamp[bounds] = chip.img[bounds]
for i in range(len(photons_list)):
if i == 0:
chip.sensor.accumulate(photons_list[i], stamp)
else:
chip.sensor.accumulate(photons_list[i], stamp, resume=True)
chip.img[bounds] = stamp[bounds]
chip.img.setOrigin(chip.bound.xmin, chip.bound.ymin)
del photons_list
del stamp
gc.collect()
return True, pos_shear
ObservationSim/MockObject/__init__.py
+ 2
- 1
View file @ fd6c3108
...@@ -3,5 +3,6 @@ from .Galaxy import Galaxy ...@@ -3,5 +3,6 @@ from .Galaxy import Galaxy
from .CatalogBase import CatalogBase from .CatalogBase import CatalogBase
from .Quasar import Quasar from .Quasar import Quasar
from .Star import Star from .Star import Star
from .Stamp import Stamp
from .SkybackgroundMap import * from .SkybackgroundMap import *
# from .CosmicRay import CosmicRay # from .CosmicRay import CosmicRay
\ No newline at end of file
ObservationSim/MockObject/_util.py
+ 13
- 2
View file @ fd6c3108
import numpy as np import numpy as np
import os import os
from scipy.interpolate import InterpolatedUnivariateSpline, interp1d from scipy.interpolate import InterpolatedUnivariateSpline, interp1d
from scipy import interpolate from scipy import interpolate, integrate
from astropy.table import Table from astropy.table import Table
import galsim import galsim
...@@ -9,6 +9,14 @@ VC_A = 2.99792458e+18 # speed of light: A/s ...@@ -9,6 +9,14 @@ VC_A = 2.99792458e+18 # speed of light: A/s
VC_M = 2.99792458e+8 # speed of light: m/s VC_M = 2.99792458e+8 # speed of light: m/s
H_PLANK = 6.626196e-27 # Plank constant: erg s H_PLANK = 6.626196e-27 # Plank constant: erg s
def comoving_dist(z, om_m=0.3111, om_L=0.6889, h=0.6766):
# Return comving distance in pc
H0 = h*100. # km / (s Mpc)
def dist_int(z):
return 1./np.sqrt(om_m*(1.+z)**3 + om_L)
res, err = integrate.quad(dist_int, 0., z)
return [res * (VC_M/1e3/H0) * 1e6, err * (VC_M/1e3/H0) * 1e6]
def magToFlux(mag): def magToFlux(mag):
""" """
flux of a given AB magnitude flux of a given AB magnitude
...@@ -447,7 +455,10 @@ def getObservedSED(sedCat, redshift=0.0, av=0.0, redden=0): ...@@ -447,7 +455,10 @@ def getObservedSED(sedCat, redshift=0.0, av=0.0, redden=0):
sw, sf = sedCat[:,0], sedCat[:,1] sw, sf = sedCat[:,0], sedCat[:,1]
# reddening # reddening
sf = reddening(sw, sf, av=av, model=redden) sf = reddening(sw, sf, av=av, model=redden)
sw, sf = sw*z, sf*(z**3) # sw, sf = sw*z, sf*(z**3)
sw, sf = sw*z, sf/z
# sw, sf = sw*z, sf
# lyman forest correction # lyman forest correction
sf = lyman_forest(sw, sf, redshift) sf = lyman_forest(sw, sf, redshift)
isedObs = (sw.copy(), sf.copy()) isedObs = (sw.copy(), sf.copy())
......
ObservationSim/ObservationSim.py
+ 106
- 145
View file @ fd6c3108
...@@ -4,6 +4,7 @@ import mpi4py.MPI as MPI ...@@ -4,6 +4,7 @@ import mpi4py.MPI as MPI
import galsim import galsim
import logging import logging
import psutil import psutil
import gc
from astropy.io import fits from astropy.io import fits
from datetime import datetime from datetime import datetime
...@@ -30,15 +31,8 @@ class Observation(object): ...@@ -30,15 +31,8 @@ class Observation(object):
self.all_filter = [] self.all_filter = []
self.Catalog = Catalog self.Catalog = Catalog
# if we want to apply field distortion?
if self.config["ins_effects"]["field_dist"] == True:
self.fd_model = FieldDistortion(fdModel_path=self.path_dict["fd_path"])
else:
self.fd_model = None
# Construct chips & filters: # Construct chips & filters:
nchips = self.focal_plane.nchip_x*self.focal_plane.nchip_y for i in range(self.focal_plane.nchips):
for i in range(nchips):
chipID = i + 1 chipID = i + 1
# Make Chip & Filter lists # Make Chip & Filter lists
...@@ -54,45 +48,32 @@ class Observation(object): ...@@ -54,45 +48,32 @@ class Observation(object):
self.filter_list.append(filt) self.filter_list.append(filt)
self.all_filter.append(filt) self.all_filter.append(filt)
# Read catalog and shear(s)
self.g1_field, self.g2_field, self.nshear = get_shear_field(config=self.config)
def run_one_chip(self, chip, filt, pointing, chip_output, wcs_fp=None, psf_model=None, shear_cat_file=None, cat_dir=None, sed_dir=None): def run_one_chip(self, chip, filt, pointing, chip_output, wcs_fp=None, psf_model=None, shear_cat_file=None, cat_dir=None, sed_dir=None):
# print(':::::::::::::::::::Current Pointing Information::::::::::::::::::') chip_output.Log_info(':::::::::::::::::::Current Pointing Information::::::::::::::::::')
# print("RA: %f, DEC; %f" % (pointing.ra, pointing.dec)) chip_output.Log_info("RA: %f, DEC; %f" % (pointing.ra, pointing.dec))
# print("Time: %s" % datetime.fromtimestamp(pointing.timestamp).isoformat()) chip_output.Log_info("Time: %s" % datetime.utcfromtimestamp(pointing.timestamp).isoformat())
# print("Exposure time: %f" % pointing.exp_time) chip_output.Log_info("Exposure time: %f" % pointing.exp_time)
# print("Satellite Position (x, y, z): (%f, %f, %f)" % (pointing.sat_x, pointing.sat_y, pointing.sat_z)) chip_output.Log_info("Satellite Position (x, y, z): (%f, %f, %f)" % (pointing.sat_x, pointing.sat_y, pointing.sat_z))
# print("Satellite Velocity (x, y, z): (%f, %f, %f)" % (pointing.sat_vx, pointing.sat_vy, pointing.sat_vz)) chip_output.Log_info("Satellite Velocity (x, y, z): (%f, %f, %f)" % (pointing.sat_vx, pointing.sat_vy, pointing.sat_vz))
# print("Position Angle: %f" % pointing.img_pa.deg) chip_output.Log_info("Position Angle: %f" % pointing.img_pa.deg)
# print('Chip : %d' % chip.chipID) chip_output.Log_info('Chip : %d' % chip.chipID)
# print(':::::::::::::::::::::::::::END:::::::::::::::::::::::::::::::::::') chip_output.Log_info(':::::::::::::::::::::::::::END:::::::::::::::::::::::::::::::::::')
chip_output.logger.info(':::::::::::::::::::Current Pointing Information::::::::::::::::::')
chip_output.logger.info("RA: %f, DEC; %f" % (pointing.ra, pointing.dec))
chip_output.logger.info("Time: %s" % datetime.fromtimestamp(pointing.timestamp).isoformat())
chip_output.logger.info("Exposure time: %f" % pointing.exp_time)
chip_output.logger.info("Satellite Position (x, y, z): (%f, %f, %f)" % (pointing.sat_x, pointing.sat_y, pointing.sat_z))
chip_output.logger.info("Satellite Velocity (x, y, z): (%f, %f, %f)" % (pointing.sat_vx, pointing.sat_vy, pointing.sat_vz))
chip_output.logger.info("Position Angle: %f" % pointing.img_pa.deg)
chip_output.logger.info('Chip : %d' % chip.chipID)
chip_output.logger.info(':::::::::::::::::::::::::::END:::::::::::::::::::::::::::::::::::')
if self.config["psf_setting"]["psf_model"] == "Gauss": if self.config["psf_setting"]["psf_model"] == "Gauss":
psf_model = PSFGauss(chip=chip, psfRa=self.config["psf_setting"]["psf_rcont"]) psf_model = PSFGauss(chip=chip, psfRa=self.config["psf_setting"]["psf_rcont"])
elif self.config["psf_setting"]["psf_model"] == "Interp": elif self.config["psf_setting"]["psf_model"] == "Interp":
psf_model = PSFInterp(chip=chip, PSF_data_file=self.path_dict["psf_dir"]) psf_model = PSFInterp(chip=chip, npsf=chip.n_psf_samples, PSF_data_file=self.path_dict["psf_dir"])
else: else:
# print("unrecognized PSF model type!!", flush=True) chip_output.Log_error("unrecognized PSF model type!!", flush=True)
chip_output.logger.error("unrecognized PSF model type!!", flush=True)
# Get (extra) shear fields # Figure out shear fields
if shear_cat_file is not None: if shear_cat_file is not None:
self.g1_field, self.g2_field, self.nshear = get_shear_field(config=self.config, shear_cat_file=shear_cat_file) self.g1_field, self.g2_field, self.nshear = get_shear_field(config=self.config, shear_cat_file=shear_cat_file)
# Apply astrometric simulation for pointing # Apply astrometric simulation for pointing
if self.config["obs_setting"]["enable_astrometric_model"]: if self.config["obs_setting"]["enable_astrometric_model"]:
dt = datetime.fromtimestamp(pointing.timestamp) dt = datetime.utcfromtimestamp(pointing.timestamp)
date_str = dt.date().isoformat() date_str = dt.date().isoformat()
time_str = dt.time().isoformat() time_str = dt.time().isoformat()
ra_cen, dec_cen = on_orbit_obs_position( ra_cen, dec_cen = on_orbit_obs_position(
...@@ -109,7 +90,7 @@ class Observation(object): ...@@ -109,7 +90,7 @@ class Observation(object):
input_vx=pointing.sat_vx, input_vx=pointing.sat_vx,
input_vy=pointing.sat_vy, input_vy=pointing.sat_vy,
input_vz=pointing.sat_vz, input_vz=pointing.sat_vz,
input_epoch="J2015.5", input_epoch="J2000",
input_date_str=date_str, input_date_str=date_str,
input_time_str=time_str input_time_str=time_str
) )
...@@ -128,24 +109,19 @@ class Observation(object): ...@@ -128,24 +109,19 @@ class Observation(object):
chip.img.wcs = wcs_fp chip.img.wcs = wcs_fp
if chip.survey_type == "photometric": if chip.survey_type == "photometric":
sky_map = None sky_map = None
# elif chip.survey_type == "spectroscopic":
# sky_map = calculateSkyMap_split_g(xLen=chip.npix_x, yLen=chip.npix_y, blueLimit=filt.blue_limit, redLimit=filt.red_limit, skyfn=self.path_dict["sky_file"], conf=chip.sls_conf, pixelSize=chip.pix_scale, isAlongY=0)
elif chip.survey_type == "spectroscopic": elif chip.survey_type == "spectroscopic":
# chip.loadSLSFLATCUBE(flat_fn='flat_cube.fits') # chip.loadSLSFLATCUBE(flat_fn='flat_cube.fits')
flat_normal = np.ones_like(chip.img.array) flat_normal = np.ones_like(chip.img.array)
if self.config["ins_effects"]["flat_fielding"] == True: if self.config["ins_effects"]["flat_fielding"] == True:
# print("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding"%chip.chipID, flush=True) chip_output.Log_info("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding"%chip.chipID)
# print(chip.img.bounds, flush=True)
chip_output.logger.info("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding"%chip.chipID)
msg = str(chip.img.bounds) msg = str(chip.img.bounds)
chip_output.logger.info(msg) chip_output.Log_info(msg)
flat_img = Effects.MakeFlatSmooth( flat_img = Effects.MakeFlatSmooth(
chip.img.bounds, chip.img.bounds,
int(self.config["random_seeds"]["seed_flat"])) int(self.config["random_seeds"]["seed_flat"]))
flat_normal = flat_normal * flat_img.array / np.mean(flat_img.array) flat_normal = flat_normal * flat_img.array / np.mean(flat_img.array)
if self.config["ins_effects"]["shutter_effect"] == True: if self.config["ins_effects"]["shutter_effect"] == True:
# print("SLS flat preprocess,CHIP %d : Apply shutter effect"%chip.chipID, flush=True) chip_output.Log_info("SLS flat preprocess,CHIP %d : Apply shutter effect"%chip.chipID)
chip_output.logger.info("SLS flat preprocess,CHIP %d : Apply shutter effect"%chip.chipID)
shuttimg = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735, shuttimg = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735,
dt=1E-3) # shutter effect normalized image for this chip dt=1E-3) # shutter effect normalized image for this chip
flat_normal = flat_normal*shuttimg flat_normal = flat_normal*shuttimg
...@@ -158,12 +134,11 @@ class Observation(object): ...@@ -158,12 +134,11 @@ class Observation(object):
pixelSize=chip.pix_scale, pixelSize=chip.pix_scale,
isAlongY=0, isAlongY=0,
flat_cube=chip.flat_cube) flat_cube=chip.flat_cube)
# sky_map = np.ones([9216, 9232])
del flat_normal del flat_normal
if pointing.pointing_type == 'MS': if pointing.pointing_type == 'MS':
# Load catalogues and templates # Load catalogues and templates
self.cat = self.Catalog(config=self.config, chip=chip, pointing=pointing, cat_dir=cat_dir, sed_dir=sed_dir, chip_output=chip_output) self.cat = self.Catalog(config=self.config, chip=chip, pointing=pointing, cat_dir=cat_dir, sed_dir=sed_dir, chip_output=chip_output, filt=filt)
chip_output.create_output_file() chip_output.create_output_file()
self.nobj = len(self.cat.objs) self.nobj = len(self.cat.objs)
...@@ -176,10 +151,33 @@ class Observation(object): ...@@ -176,10 +151,33 @@ class Observation(object):
if temp_filter.filter_type.lower() == self.config["obs_setting"]["cut_in_band"].lower(): if temp_filter.filter_type.lower() == self.config["obs_setting"]["cut_in_band"].lower():
cut_filter = temp_filter cut_filter = temp_filter
if self.config["ins_effects"]["field_dist"] == True:
self.fd_model = FieldDistortion(chip=chip)
else:
self.fd_model = None
# Loop over objects # Loop over objects
missed_obj = 0 missed_obj = 0
bright_obj = 0 bright_obj = 0
dim_obj = 0 dim_obj = 0
h_ext = generateExtensionHeader(
chip=chip,
xlen=chip.npix_x,
ylen=chip.npix_y,
ra=pointing.ra,
dec=pointing.dec,
pa=pointing.img_pa.deg,
gain=chip.gain,
readout=chip.read_noise,
dark=chip.dark_noise,
saturation=90000,
pixel_scale=chip.pix_scale,
pixel_size=chip.pix_size,
xcen=chip.x_cen,
ycen=chip.y_cen,
extName='raw')
for j in range(self.nobj): for j in range(self.nobj):
# (DEBUG) # (DEBUG)
...@@ -187,61 +185,48 @@ class Observation(object): ...@@ -187,61 +185,48 @@ class Observation(object):
# break # break
obj = self.cat.objs[j] obj = self.cat.objs[j]
if obj.type == 'star' and self.config["run_option"]["galaxy_only"]:
continue
elif obj.type == 'galaxy' and self.config["run_option"]["star_only"]:
continue
elif obj.type == 'quasar' and self.config["run_option"]["star_only"]:
continue
# load and convert SED; also caculate object's magnitude in all CSST bands # load and convert SED; also caculate object's magnitude in all CSST bands
try: try:
sed_data = self.cat.load_sed(obj) sed_data = self.cat.load_sed(obj)
norm_filt = self.cat.load_norm_filt(obj) norm_filt = self.cat.load_norm_filt(obj)
obj.sed, obj.param["mag_%s"%filt.filter_type], obj.param["flux_%s"%filt.filter_type] = self.cat.convert_sed( obj.sed, obj.param["mag_%s"%filt.filter_type.lower()], obj.param["flux_%s"%filt.filter_type.lower()] = self.cat.convert_sed(
mag=obj.param["mag_use_normal"], mag=obj.param["mag_use_normal"],
sed=sed_data, sed=sed_data,
target_filt=filt, target_filt=filt,
norm_filt=norm_filt, norm_filt=norm_filt,
) )
_, obj.param["mag_%s"%cut_filter.filter_type], obj.param["flux_%s"%cut_filter.filter_type] = self.cat.convert_sed( _, obj.param["mag_%s"%cut_filter.filter_type.lower()], obj.param["flux_%s"%cut_filter.filter_type.lower()] = self.cat.convert_sed(
mag=obj.param["mag_use_normal"], mag=obj.param["mag_use_normal"],
sed=sed_data, sed=sed_data,
target_filt=cut_filter, target_filt=cut_filter,
norm_filt=norm_filt, norm_filt=norm_filt,
) )
except Exception as e: except Exception as e:
# print(e)
traceback.print_exc() traceback.print_exc()
chip_output.logger.error(e) chip_output.Log_error(e)
continue continue
# chip_output.logger.info("debug point #1") # [TODO] Testing
# chip_output.Log_info("mag_%s = %.3f"%(filt.filter_type.lower(), obj.param["mag_%s"%filt.filter_type.lower()]))
# Exclude very bright/dim objects (for now) # Exclude very bright/dim objects (for now)
# if filt.is_too_bright(mag=obj.getMagFilter(filt)):
# if filt.is_too_bright(mag=obj.mag_use_normal):
if cut_filter.is_too_bright( if cut_filter.is_too_bright(
mag=obj.param["mag_%s"%self.config["obs_setting"]["cut_in_band"].lower()], mag=obj.param["mag_%s"%self.config["obs_setting"]["cut_in_band"].lower()],
margin=self.config["obs_setting"]["mag_sat_margin"]): margin=self.config["obs_setting"]["mag_sat_margin"]):
# print("obj too birght!!", flush=True) chip_output.Log_info("obj %s too birght!! mag_%s = %.3f"%(obj.id, cut_filter.filter_type, obj.param["mag_%s"%self.config["obs_setting"]["cut_in_band"].lower()]))
if obj.type != 'galaxy': bright_obj += 1
bright_obj += 1 obj.unload_SED()
obj.unload_SED() continue
continue
if filt.is_too_dim( if filt.is_too_dim(
mag=obj.getMagFilter(filt), mag=obj.getMagFilter(filt),
margin=self.config["obs_setting"]["mag_lim_margin"]): margin=self.config["obs_setting"]["mag_lim_margin"]):
# if cut_filter.is_too_dim(mag=obj.param["mag_%s"%self.config["obs_setting"]["cut_in_band"].lower()]): chip_output.Log_info("obj %s too dim!! mag_%s = %.3f"%(obj.id, filt.filter_type, obj.getMagFilter(filt)))
# print("obj too dim!!", flush=True)
dim_obj += 1 dim_obj += 1
obj.unload_SED() obj.unload_SED()
# print(obj.getMagFilter(filt))
continue continue
# chip_output.logger.info("debug point #2") # Get corresponding shear values
if self.config["shear_setting"]["shear_type"] == "constant": if self.config["shear_setting"]["shear_type"] == "constant":
if obj.type == 'star': if obj.type == 'star':
obj.g1, obj.g2 = 0., 0. obj.g1, obj.g2 = 0., 0.
...@@ -249,52 +234,34 @@ class Observation(object): ...@@ -249,52 +234,34 @@ class Observation(object):
obj.g1, obj.g2 = self.g1_field, self.g2_field obj.g1, obj.g2 = self.g1_field, self.g2_field
elif self.config["shear_setting"]["shear_type"] == "extra": elif self.config["shear_setting"]["shear_type"] == "extra":
try: try:
# TODO: every object with individual shear from input catalog(s) # [TODO]: every object with individual shear from input catalog(s)
obj.g1, obj.g2 = self.g1_field[j], self.g2_field[j] obj.g1, obj.g2 = self.g1_field[j], self.g2_field[j]
except: except:
# print("failed to load external shear.") chip_output.Log_error("failed to load external shear.")
chip_output.logger.error("failed to load external shear.")
pass
# chip_output.logger.info("debug point #3")
elif self.config["shear_setting"]["shear_type"] == "catalog": elif self.config["shear_setting"]["shear_type"] == "catalog":
pass pass
else: else:
chip_output.logger.error("Unknown shear input") chip_output.Log_error("Unknown shear input")
raise ValueError("Unknown shear input") raise ValueError("Unknown shear input")
# chip_output.logger.info("debug point #4") # Get position of object on the focal plane
header_wcs = generateExtensionHeader( pos_img, offset, local_wcs, real_wcs, fd_shear = obj.getPosImg_Offset_WCS(img=chip.img, fdmodel=self.fd_model, chip=chip, verbose=False, img_header=h_ext)
xlen=chip.npix_x,
ylen=chip.npix_y, # [TODO] For now, only consider objects which their centers (after field distortion) are projected within the focal plane
ra=ra_cen, # Otherwise they will be considered missed objects
dec=dec_cen, # if pos_img.x == -1 or pos_img.y == -1 or (not chip.isContainObj(x_image=pos_img.x, y_image=pos_img.y, margin=0.)):
pa=pointing.img_pa.deg,
gain=chip.gain,
readout=chip.read_noise,
dark=chip.dark_noise,
saturation=90000,
psize=chip.pix_scale,
row_num=chip.rowID,
col_num=chip.colID,
extName='raw')
pos_img, offset, local_wcs, real_wcs = obj.getPosImg_Offset_WCS(img=chip.img, fdmodel=self.fd_model, chip=chip, verbose=False, img_header=header_wcs)
if pos_img.x == -1 or pos_img.y == -1: if pos_img.x == -1 or pos_img.y == -1:
# Exclude object which is outside the chip area (after field distortion) chip_output.Log_info('obj_ra = %.6f, obj_dec = %.6f, obj_ra_orig = %.6f, obj_dec_orig = %.6f'%(obj.ra, obj.dec, obj.ra_orig, obj.dec_orig))
# print("obj missed!!") chip_output.Log_error("Objected missed: %s"%(obj.id))
missed_obj += 1 missed_obj += 1
obj.unload_SED() obj.unload_SED()
continue continue
# chip_output.logger.info("debug point #5")
# Draw object & update output catalog # Draw object & update output catalog
try: try:
# chip_output.logger.info("debug point #6")
# chip_output.logger.info("current filter type: %s"%filt.filter_type)
if self.config["run_option"]["out_cat_only"]: if self.config["run_option"]["out_cat_only"]:
isUpdated = True isUpdated = True
obj.real_pos = obj.getRealPos(chip.img, global_x=obj.posImg.x, global_y=obj.posImg.y, img_real_wcs=obj.real_wcs)
pos_shear = 0. pos_shear = 0.
elif chip.survey_type == "photometric" and not self.config["run_option"]["out_cat_only"]: elif chip.survey_type == "photometric" and not self.config["run_option"]["out_cat_only"]:
isUpdated, pos_shear = obj.drawObj_multiband( isUpdated, pos_shear = obj.drawObj_multiband(
...@@ -306,8 +273,9 @@ class Observation(object): ...@@ -306,8 +273,9 @@ class Observation(object):
chip=chip, chip=chip,
g1=obj.g1, g1=obj.g1,
g2=obj.g2, g2=obj.g2,
exptime=pointing.exp_time exptime=pointing.exp_time,
) fd_shear=fd_shear)
elif chip.survey_type == "spectroscopic" and not self.config["run_option"]["out_cat_only"]: elif chip.survey_type == "spectroscopic" and not self.config["run_option"]["out_cat_only"]:
isUpdated, pos_shear = obj.drawObj_slitless( isUpdated, pos_shear = obj.drawObj_slitless(
tel=self.tel, tel=self.tel,
...@@ -319,30 +287,38 @@ class Observation(object): ...@@ -319,30 +287,38 @@ class Observation(object):
g1=obj.g1, g1=obj.g1,
g2=obj.g2, g2=obj.g2,
exptime=pointing.exp_time, exptime=pointing.exp_time,
normFilter=norm_filt) normFilter=norm_filt,
# chip_output.logger.info("debug point #7") fd_shear=fd_shear)
if isUpdated:
if isUpdated == 1:
# TODO: add up stats # TODO: add up stats
# print("updating output catalog...")
chip_output.cat_add_obj(obj, pos_img, pos_shear) chip_output.cat_add_obj(obj, pos_img, pos_shear)
pass pass
elif isUpdated == 0:
missed_obj += 1
chip_output.Log_error("Objected missed: %s"%(obj.id))
else: else:
# print("object omitted", flush=True) chip_output.Log_error("Draw error, object omitted: %s"%(obj.id))
continue continue
except Exception as e: except Exception as e:
# print(e)
traceback.print_exc() traceback.print_exc()
chip_output.logger.error(e) chip_output.Log_error(e)
pass
# # [C6 TEST]
# chip_output.Log_info("check running:1: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
# chip_output.Log_info('draw object %s'%obj.id)
# chip_output.Log_info('mag = %.3f'%obj.param['mag_use_normal'])
# Unload SED: # Unload SED:
obj.unload_SED() obj.unload_SED()
del obj del obj
gc.collect()
del psf_model del psf_model
del self.cat del self.cat
gc.collect()
# print("check running:1: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True) chip_output.Log_info("check running:1: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
chip_output.logger.info("check running:1: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
# Detector Effects # Detector Effects
# =========================================================== # ===========================================================
...@@ -357,6 +333,7 @@ class Observation(object): ...@@ -357,6 +333,7 @@ class Observation(object):
ra_cen=pointing.ra, ra_cen=pointing.ra,
dec_cen=pointing.dec, dec_cen=pointing.dec,
img_rot=pointing.img_pa, img_rot=pointing.img_pa,
exptime=pointing.exp_time,
pointing_ID=pointing.id, pointing_ID=pointing.id,
timestamp_obs=pointing.timestamp, timestamp_obs=pointing.timestamp,
pointing_type=pointing.pointing_type, pointing_type=pointing.pointing_type,
...@@ -364,7 +341,7 @@ class Observation(object): ...@@ -364,7 +341,7 @@ class Observation(object):
logger=chip_output.logger) logger=chip_output.logger)
if pointing.pointing_type == 'MS': if pointing.pointing_type == 'MS':
datetime_obs = datetime.fromtimestamp(pointing.timestamp) datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
date_obs = datetime_obs.strftime("%y%m%d") date_obs = datetime_obs.strftime("%y%m%d")
time_obs = datetime_obs.strftime("%H%M%S") time_obs = datetime_obs.strftime("%H%M%S")
h_prim = generatePrimaryHeader( h_prim = generatePrimaryHeader(
...@@ -373,45 +350,30 @@ class Observation(object): ...@@ -373,45 +350,30 @@ class Observation(object):
pointNum = str(pointing.id), pointNum = str(pointing.id),
ra=pointing.ra, ra=pointing.ra,
dec=pointing.dec, dec=pointing.dec,
psize=chip.pix_scale, pixel_scale=chip.pix_scale,
row_num=chip.rowID,
col_num=chip.colID,
date=date_obs, date=date_obs,
time_obs=time_obs, time_obs=time_obs,
exptime=pointing.exp_time, exptime=pointing.exp_time,
im_type='SCI', im_type='SCI',
sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z], sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z],
sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz]) sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz],
h_ext = generateExtensionHeader( chip_name=str(chip.chipID).rjust(2, '0'))
xlen=chip.npix_x,
ylen=chip.npix_y,
ra=pointing.ra,
dec=pointing.dec,
pa=pointing.img_pa.deg,
gain=chip.gain,
readout=chip.read_noise,
dark=chip.dark_noise,
saturation=90000,
psize=chip.pix_scale,
row_num=chip.rowID,
col_num=chip.colID,
extName='raw')
chip.img = galsim.Image(chip.img.array, dtype=np.uint16) chip.img = galsim.Image(chip.img.array, dtype=np.uint16)
hdu1 = fits.PrimaryHDU(header=h_prim) hdu1 = fits.PrimaryHDU(header=h_prim)
hdu1.add_checksum()
hdu2 = fits.ImageHDU(chip.img.array, header=h_ext) hdu2 = fits.ImageHDU(chip.img.array, header=h_ext)
hdu2.add_checksum()
hdu1 = fits.HDUList([hdu1, hdu2]) hdu1 = fits.HDUList([hdu1, hdu2])
fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits') fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits')
hdu1.writeto(fname, output_verify='ignore', overwrite=True) hdu1.writeto(fname, output_verify='ignore', overwrite=True)
# print("# objects that are too bright %d out of %d"%(bright_obj, self.nobj))
# print("# objects that are too dim %d out of %d"%(dim_obj, self.nobj)) chip_output.Log_info("# objects that are too bright %d out of %d"%(bright_obj, self.nobj))
# print("# objects that are missed %d out of %d"%(missed_obj, self.nobj)) chip_output.Log_info("# objects that are too dim %d out of %d"%(dim_obj, self.nobj))
chip_output.logger.info("# objects that are too bright %d out of %d"%(bright_obj, self.nobj)) chip_output.Log_info("# objects that are missed %d out of %d"%(missed_obj, self.nobj))
chip_output.logger.info("# objects that are too dim %d out of %d"%(dim_obj, self.nobj))
chip_output.logger.info("# objects that are missed %d out of %d"%(missed_obj, self.nobj))
del chip.img del chip.img
# print("check running:2: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True) chip_output.Log_info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
chip_output.logger.info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
def runExposure_MPI_PointingList(self, pointing_list, shear_cat_file=None, chips=None, use_mpi=False): def runExposure_MPI_PointingList(self, pointing_list, shear_cat_file=None, chips=None, use_mpi=False):
if use_mpi: if use_mpi:
...@@ -450,7 +412,7 @@ class Observation(object): ...@@ -450,7 +412,7 @@ class Observation(object):
chip = run_chips[ichip] chip = run_chips[ichip]
filt = run_filts[ichip] filt = run_filts[ichip]
# print("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid), flush=True) # chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid))
chip_output = ChipOutput( chip_output = ChipOutput(
config=self.config, config=self.config,
focal_plane=self.focal_plane, focal_plane=self.focal_plane,
...@@ -461,14 +423,13 @@ class Observation(object): ...@@ -461,14 +423,13 @@ class Observation(object):
pointing_ID=pointing_ID, pointing_ID=pointing_ID,
subdir=sub_img_dir, subdir=sub_img_dir,
prefix=prefix) prefix=prefix)
chip_output.logger.info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid)) chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid))
self.run_one_chip( self.run_one_chip(
chip=chip, chip=chip,
filt=filt, filt=filt,
chip_output=chip_output, chip_output=chip_output,
pointing=pointing, pointing=pointing)
cat_dir=self.path_dict["cat_dir"]) chip_output.Log_info("finished running chip#%d..."%(chip.chipID))
print("finished running chip#%d..."%(chip.chipID), flush=True)
chip_output.logger.info("finished running chip#%d..."%(chip.chipID))
for handler in chip_output.logger.handlers[:]: for handler in chip_output.logger.handlers[:]:
chip_output.logger.removeHandler(handler) chip_output.logger.removeHandler(handler)
gc.collect()
ObservationSim/PSF/FieldDistortion.py
+ 61
- 26
View file @ fd6c3108
...@@ -3,31 +3,46 @@ import numpy as np ...@@ -3,31 +3,46 @@ import numpy as np
class FieldDistortion(object): class FieldDistortion(object):
def __init__(self, fdModel=None, fdModel_path=None): def __init__(self, chip, fdModel=None, fdModel_path=None):
if fdModel is None: if fdModel is None:
import pickle if hasattr(chip, 'fdModel'):
if fdModel_path is not None: self.fdModel = chip.fdModel
elif fdModel_path is not None:
import pickle
with open(fdModel_path, "rb") as f: with open(fdModel_path, "rb") as f:
self.fdModel = pickle.load(f) self.fdModel = pickle.load(f)
else: else:
# with open("/data/simudata/CSSOSDataProductsSims/data/FieldDistModelv1.0.pickle", "rb") as f: raise ValueError("Error: no field distortion model has been specified!")
with open("/data/simudata/CSSOSDataProductsSims/data/FieldDistModelv2.0.pickle", "rb") as f:
self.fdModel = pickle.load(f)
else: else:
self.fdModel = fdModel self.fdModel = fdModel
self.ifdModel = self.fdModel["wave1"]
self.ixfdModel = self.ifdModel["xImagePos"]
self.iyfdModel = self.ifdModel["yImagePos"]
# first-order derivatives of the global field distortion model
self.ifx_dx = self.ixfdModel.partial_derivative(1,0)
self.ifx_dy = self.ixfdModel.partial_derivative(0,1)
self.ify_dx = self.iyfdModel.partial_derivative(1,0)
self.ify_dy = self.iyfdModel.partial_derivative(0,1)
if "residual" in self.fdModel["wave1"]:
self.irsModel = self.fdModel["wave1"]["residual"]["ccd" + chip.getChipLabel(chipID=chip.chipID)]
self.ixrsModel = self.irsModel["xResidual"]
self.iyrsModel = self.irsModel["yResidual"]
# first-order derivatives of the residual field distortion model
self.irx_dx = self.ixrsModel.partial_derivative(1,0)
self.irx_dy = self.ixrsModel.partial_derivative(0,1)
self.iry_dx = self.iyrsModel.partial_derivative(1,0)
self.iry_dy = self.iyrsModel.partial_derivative(0,1)
else:
self.irsModel = None
def isContainObj_FD(self, chip, pos_img): def isContainObj_FD(self, chip, pos_img):
# ifdModel = self.fdModel["ccd" + chip.getChipLabel(chipID=chip.chipID)]["wave1"] xLowI, xUpI, yLowI, yUpI = self.ifdModel["interpLimit"]
# x, y = pos_img.x, pos_img.y
# xLowI, xUpI, yLowI, yUpI = ifdModel["InterpLim"]
ifdModel = self.fdModel["wave1"]
xLowI, xUpI, yLowI, yUpI = ifdModel["interpLimit"]
x, y = pos_img.x, pos_img.y x, y = pos_img.x, pos_img.y
if (xLowI - x)*(xUpI - x) > 0 or (yLowI - y) * (yUpI - y) > 0: if (xLowI - x) * (xUpI - x) > 0 or (yLowI - y) * (yUpI - y) > 0:
return False return False
return True return True
def get_Distorted(self, chip, pos_img, bandpass=None): def get_distorted(self, chip, pos_img, bandpass=None):
""" Get the distored position for an undistorted image position """ Get the distored position for an undistorted image position
Parameters: Parameters:
...@@ -42,19 +57,39 @@ class FieldDistortion(object): ...@@ -42,19 +57,39 @@ class FieldDistortion(object):
the distored position. the distored position.
""" """
if not self.isContainObj_FD(chip=chip, pos_img=pos_img): if not self.isContainObj_FD(chip=chip, pos_img=pos_img):
return galsim.PositionD(-1, -1) return galsim.PositionD(-1, -1), None
# ifdModel = self.fdModel["ccd" + chip.getChipLabel(chipID=chip.chipID)]["wave1"]
ifdModel = self.fdModel["wave1"]
irsModel = self.fdModel["wave1"]["residual"]["ccd" + chip.getChipLabel(chipID=chip.chipID)]
x, y = pos_img.x, pos_img.y x, y = pos_img.x, pos_img.y
x = ifdModel["xImagePos"](x, y)[0][0] x = self.ixfdModel(x, y)[0][0]
y = ifdModel["yImagePos"](x, y)[0][0] y = self.iyfdModel(x, y)[0][0]
x1LowI, x1UpI, y1LowI, y1UpI = irsModel["interpLimit"] ix_dx = self.ifx_dx(x, y)
if (x1LowI-x)*(x1UpI-x) <=0 and (y1LowI-y)*(y1UpI-y)<=0: ix_dy = self.ifx_dy(x, y)
dx = irsModel["xResidual"](x, y)[0][0] iy_dx = self.ify_dx(x, y)
dy = irsModel["yResidual"](x, y)[0][0] iy_dy = self.ify_dy(x, y)
if self.irsModel is not None:
# x1LowI, x1UpI, y1LowI, y1UpI = self.irsModel["interpLimit"]
# if (x1LowI-x)*(x1UpI-x) <=0 and (y1LowI-y)*(y1UpI-y)<=0:
# dx = self.ixrsModel(x, y)[0][0]
# dy = self.iyrsModel(x, y)[0][0]
# x += dx
# y += dy
# # field distortion induced ellipticity
# ix_dx = self.ifx_dx(x, y) + self.irx_dx(x, y)
# ix_dy = self.ifx_dy(x, y) + self.irx_dy(x, y)
# iy_dx = self.ify_dx(x, y) + self.iry_dx(x, y)
# iy_dy = self.ify_dy(x, y) + self.iry_dy(x, y)
# else:
# return galsim.PositionD(-1, -1), None
dx = self.ixrsModel(x, y)[0][0]
dy = self.iyrsModel(x, y)[0][0]
x += dx x += dx
y += dy y += dy
else: # field distortion induced ellipticity
return galsim.PositionD(-1, -1) ix_dx = self.ifx_dx(x, y) + self.irx_dx(x, y)
return galsim.PositionD(x, y) ix_dy = self.ifx_dy(x, y) + self.irx_dy(x, y)
iy_dx = self.ify_dx(x, y) + self.iry_dx(x, y)
iy_dy = self.ify_dy(x, y) + self.iry_dy(x, y)
g1k_fd = 0.0 + (iy_dy - ix_dx) / (iy_dy + ix_dx)
g2k_fd = 0.0 - (iy_dx + ix_dy) / (iy_dy + ix_dx)
fd_shear = galsim.Shear(g1=g1k_fd, g2=g2k_fd)
return galsim.PositionD(x, y), fd_shear
ObservationSim/PSF/PSFInterp.py
+ 6
- 3
View file @ fd6c3108
...@@ -216,6 +216,7 @@ class PSFInterp(PSFModel): ...@@ -216,6 +216,7 @@ class PSFInterp(PSFModel):
self.sigGauss = psfRa self.sigGauss = psfRa
self.iccd = int(chip.getChipLabel(chipID=chip.chipID)) self.iccd = int(chip.getChipLabel(chipID=chip.chipID))
# self.iccd = chip.chip_name
if PSF_data_file == None: if PSF_data_file == None:
print('Error - PSF_data_file is None') print('Error - PSF_data_file is None')
sys.exit() sys.exit()
...@@ -256,7 +257,8 @@ class PSFInterp(PSFModel): ...@@ -256,7 +257,8 @@ class PSFInterp(PSFModel):
def _getPSFwave(self, iccd, PSF_data_file, PSF_data_prefix): def _getPSFwave(self, iccd, PSF_data_file, PSF_data_prefix):
fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_ccd{:}.h5'.format(iccd), 'r') # fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_ccd{:}.h5'.format(iccd), 'r')
fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_{:}.h5'.format(iccd), 'r')
nwave = len(fq.keys()) nwave = len(fq.keys())
fq.close() fq.close()
return nwave return nwave
...@@ -264,7 +266,8 @@ class PSFInterp(PSFModel): ...@@ -264,7 +266,8 @@ class PSFInterp(PSFModel):
def _loadPSF(self, iccd, PSF_data_file, PSF_data_prefix): def _loadPSF(self, iccd, PSF_data_file, PSF_data_prefix):
psfSet = [] psfSet = []
fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_ccd{:}.h5'.format(iccd), 'r') # fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_ccd{:}.h5'.format(iccd), 'r')
fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_{:}.h5'.format(iccd), 'r')
for ii in range(self.nwave): for ii in range(self.nwave):
iwave = ii+1 iwave = ii+1
psfWave = [] psfWave = []
...@@ -322,7 +325,7 @@ class PSFInterp(PSFModel): ...@@ -322,7 +325,7 @@ class PSFInterp(PSFModel):
""" """
pixSize = np.rad2deg(self.pixsize*1e-3/28)*3600 #set psf pixsize pixSize = np.rad2deg(self.pixsize*1e-3/28)*3600 #set psf pixsize
assert self.iccd == int(chip.getChipLabel(chipID=chip.chipID)), 'ERROR: self.iccd != chip.chipID' # assert self.iccd == int(chip.getChipLabel(chipID=chip.chipID)), 'ERROR: self.iccd != chip.chipID'
twave = self._findWave(bandpass) twave = self._findWave(bandpass)
if twave == -1: if twave == -1:
print("!!!PSF bandpass does not match.") print("!!!PSF bandpass does not match.")
......
ObservationSim/_util.py
+ 12
- 8
View file @ fd6c3108
...@@ -12,10 +12,11 @@ def parse_args(): ...@@ -12,10 +12,11 @@ def parse_args():
can be set in the .yaml config file as well. can be set in the .yaml config file as well.
''' '''
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('config_file', help='.yaml config file for simulation settings.') parser.add_argument('--config_file', type=str, required=True, help='.yaml config file for simulation settings.')
parser.add_argument('-c', '--config_dir', help='Directory that houses the .yaml config file.') parser.add_argument('--catalog', type=str, required=True, help='name of the catalog interface class to be loaded.')
parser.add_argument('-d', '--data_dir', help='Directory that houses the input data.') parser.add_argument('-c', '--config_dir', type=str, help='Directory that houses the .yaml config file.')
parser.add_argument('-w', '--work_dir', help='The path for output.') parser.add_argument('-d', '--data_dir', type=str, help='Directory that houses the input data.')
parser.add_argument('-w', '--work_dir', type=str, help='The path for output.')
return parser.parse_args() return parser.parse_args()
def generate_pointing_list(config, pointing_filename=None, data_dir=None): def generate_pointing_list(config, pointing_filename=None, data_dir=None):
...@@ -144,7 +145,7 @@ def makeSubDir_PointingList(path_dict, config, pointing_ID=0): ...@@ -144,7 +145,7 @@ def makeSubDir_PointingList(path_dict, config, pointing_ID=0):
return subImgdir, prefix return subImgdir, prefix
def get_shear_field(config, shear_cat_file=None): def get_shear_field(config, shear_cat_file=None):
if not config["shear_setting"]["shear_type"] in ["constant", "extra"]: if not config["shear_setting"]["shear_type"] in ["constant", "extra", "catalog"]:
raise ValueError("Please set a right 'shear_method' parameter.") raise ValueError("Please set a right 'shear_method' parameter.")
if config["shear_setting"]["shear_type"] == "constant": if config["shear_setting"]["shear_type"] == "constant":
...@@ -152,16 +153,19 @@ def get_shear_field(config, shear_cat_file=None): ...@@ -152,16 +153,19 @@ def get_shear_field(config, shear_cat_file=None):
g2 = config["shear_setting"]["reduced_g2"] g2 = config["shear_setting"]["reduced_g2"]
nshear = 1 nshear = 1
# TODO logging # TODO logging
else: elif config["shear_setting"]["shear_type"] == "extra":
# TODO logging # TODO logging
if not os.path.exists(shear_cat_file): if not os.path.exists(shear_cat_file):
raise ValueError("Cannot find shear catalog file.") raise ValueError("Cannot find external shear catalog file.")
try: try:
shearCat = np.loadtxt(shear_cat_file) shearCat = np.loadtxt(shear_cat_file)
nshear = shearCat.shape[0] nshear = shearCat.shape[0]
g1, g2 = shearCat[:, 0], shearCat[:, 1] g1, g2 = shearCat[:, 0], shearCat[:, 1]
except: except:
print("Failed to the shear catalog file.") print("Failed to read the shear catalog file.")
print("Setting to no shear.") print("Setting to no shear.")
g1, g2 = 0., 0. g1, g2 = 0., 0.
else:
g1, g2 = 0., 0.
nshear = 0
return g1, g2, nshear return g1, g2, nshear
\ No newline at end of file
README.md
+ 14
- 6
View file @ fd6c3108
...@@ -2,24 +2,32 @@ ...@@ -2,24 +2,32 @@
## 重要更新或问题修复： ## 重要更新或问题修复：
* 2023.05.01: 更新至v2.0版本，新功能包含：
* C6仿真实现
* fits贴图功能
* 焦面探测器属性单独定义
* 无缝光谱探测器一度旋转
* 像场畸导致目标形变的仿真
* 导星探测器仿真
* 仪器参数更新，bug修复等
* 2022.08.23: 更新至最新的透过率曲线，在图像头文件中加入版本号。仿真软件版本号进行重新标记，当前版本为1.0.4 * 2022.08.23: 更新至最新的透过率曲线，在图像头文件中加入版本号。仿真软件版本号进行重新标记，当前版本为1.0.4
* 2022.06.20: 新版本v0.5上线，请于release_v0.5 branch中进行clone操作(注：目前已重新设置为v1.0，release_v1.0.0) * 2022.06.20: 版本v0.5上线，请于release_v0.5 branch中进行clone操作(注：目前已重新设置为v1.0，release_v1.0.0)
* 2022.04.13: 修复了天测模块中的调用错误 * 2022.04.13: 修复了天测模块中的调用错误
* 2022.04.08: 修复了仅输出星表选项("out_cat_only")存在的问题 * 2022.04.08: 修复了仅输出星表选项("out_cat_only")存在的问题
## 使用方法及相关说明 ## 使用方法及相关说明
* 软件安装和使用方法可参考：[软件说明文档](https://kdocs.cn/l/ckrtHibdV3OS)。目前还较为粗糙，持续更新中 * 软件安装和使用方法可参考：[软件说明文档](https://kdocs.cn/l/ckrtHibdV3OS)。目前还较为粗糙，持续更新中
<!---
* 仿真数据说明文档（见以上代码目录）：CSST主巡天Cycle5模拟数据说明.pdf * 仿真数据说明文档：[C6数据说明文档](https://www.kdocs.cn/l/cuRmPf71Ly7v)
-->
* 在线版使用说明：[CSST网页接口使用说明](https://kdocs.cn/l/cl81flAXe1YQ) * 在线版使用说明：[CSST网页接口使用说明](https://kdocs.cn/l/cl81flAXe1YQ)
* 软件安装及问题反馈表：[问题反馈表]（https://f.kdocs.cn/w/XYcBkE63/） * 软件安装及问题反馈表：[问题反馈表](https://f.kdocs.cn/w/XYcBkE63/)
## 相关数据 ## 相关数据
* PSF采样及像场畸变数据下载：[百度云盘](https://pan.baidu.com/s/1oK2rwt9B1MVy1tyD3vx_gw)，密码：i7yq * PSF采样数据下载：[百度云盘](https://pan.baidu.com/s/1_2gbO6-d7x001_FMqZ1tfw)，密码：2awt
* 滤光片及无缝光谱相关数据：[百度云盘](https://pan.baidu.com/s/1JROggg8IKf27i1DnwTaPAw)，密码：07v5 * 滤光片及无缝光谱相关数据：[百度云盘](https://pan.baidu.com/s/1JROggg8IKf27i1DnwTaPAw)，密码：07v5
......
config/config_NGP_dev.yaml → config/config_C6.yaml
+ 61
- 63
View file @ fd6c3108
...@@ -2,24 +2,16 @@ ...@@ -2,24 +2,16 @@
############################################### ###############################################
# #
# Configuration file for CSST simulation # Configuration file for CSST simulation
# CSST-Sim Group, 2021/10/07 # CSST-Sim Group, 2023/04/25
# #
############################################### ###############################################
# Base diretories and naming setup # Base diretories and naming setup
# Can add some of the command-line arguments here as well; # Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent # OK to pass either way or both, as long as they are consistent
# work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/" work_dir: "/share/home/fangyuedong/fgs_sim/csst-simulation/workplace/"
work_dir: "/share/home/fangyuedong/test_psf_rot/csst-simulation/workplace/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/" data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "PSF_TEST" run_name: "C6_spectroscopic_20230225"
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_test_NGP_2.17.dat"
# Whether to use MPI # Whether to use MPI
run_option: run_option:
...@@ -33,12 +25,35 @@ run_option: ...@@ -33,12 +25,35 @@ run_option:
# If yes, no imaging simulation will run # If yes, no imaging simulation will run
out_cat_only: NO out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_C6_20221212"
star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
galaxy_cat: "cat2CSSTSim_bundle/"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Catalog_C6_20221212/sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
# Only simulate stars? # Only simulate stars?
star_only: YES star_only: NO
# Only simulate galaxies? # Only simulate galaxies?
galaxy_only: NO galaxy_only: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
############################################### ###############################################
# Observation setting # Observation setting
############################################### ###############################################
...@@ -47,14 +62,14 @@ obs_setting: ...@@ -47,14 +62,14 @@ obs_setting:
# Options for survey types: # Options for survey types:
# "Photometric": simulate photometric chips only # "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only # "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane # "All": simulate full focal plane
survey_type: "All" survey_type: "Photometric"
# Exposure time [seconds] # Exposure time [seconds]
exp_time: 150. exp_time: 150.
# Observation starting date & time # Observation starting date & time
# (Subject to change)
date_obs: "210525" # [yymmdd] date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss] time_obs: "120000" # [hhmmss]
...@@ -62,10 +77,16 @@ obs_setting: ...@@ -62,10 +77,16 @@ obs_setting:
# Note: NOT valid when a pointing list file is specified # Note: NOT valid when a pointing list file is specified
ra_center: 192.8595 ra_center: 192.8595
dec_center: 27.1283 dec_center: 27.1283
# Image rotation [degree] # Image rotation [degree]
image_rot: -113.4333 image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_radec_246.5_40.dat"
# Number of calibration pointings # Number of calibration pointings
np_cal: 0 np_cal: 0
...@@ -79,14 +100,13 @@ obs_setting: ...@@ -79,14 +100,13 @@ obs_setting:
# - give a list of indexes of chips: [ip_1, ip_2...] # - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null # - run all chips: null
# Note: for all pointings # Note: for all pointings
run_chips: null run_chips: [24]
# Whether to enable astrometric modeling # Whether to enable astrometric modeling
# astrometric_lib: "libshao.so"
enable_astrometric_model: True enable_astrometric_model: True
# Cut by saturation/limiting magnitude in which band? # Cut by saturation magnitude in which band?
cut_in_band: "g" cut_in_band: "z"
# saturation magnitude margin # saturation magnitude margin
mag_sat_margin: -2.5 mag_sat_margin: -2.5
...@@ -94,20 +114,6 @@ obs_setting: ...@@ -94,20 +114,6 @@ obs_setting:
# limiting magnitude margin # limiting magnitude margin
mag_lim_margin: +1.0 mag_lim_margin: +1.0
###############################################
# Input path setting
###############################################
# Default path settings for WIDE survey simulation
input_path:
cat_dir: "OnOrbitCalibration/CTargets20211231"
star_cat: "CT-NGP_r1.8_G28.hdf5"
galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Templates/Galaxy/"
############################################### ###############################################
# PSF setting # PSF setting
############################################### ###############################################
...@@ -125,13 +131,7 @@ psf_setting: ...@@ -125,13 +131,7 @@ psf_setting:
# path to PSF data # path to PSF data
# NOTE: only valid for "Interp" PSF # NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube" psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
# path to field-distortion model
# Note: only valid when ins_effects: field_dist is "ON"
fd_path: "FieldDistModelGlobal_v1.0.pickle"
# sigma_spin: 0.0 # psf spin?
############################################### ###############################################
# Shear setting # Shear setting
...@@ -140,19 +140,13 @@ psf_setting: ...@@ -140,19 +140,13 @@ psf_setting:
shear_setting: shear_setting:
# Options to generate mock shear field: # Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear # "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog (not available yet) # "catalog": from catalog
# "extra": from seprate file # "extra": from seprate file
shear_type: "constant" shear_type: "catalog"
# For constant shear filed # For constant shear filed
reduced_g1: 0.026 reduced_g1: 0.
reduced_g2: 0.015 reduced_g2: 0.
# Representation of the shear vector?
reShear: "E"
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
# Extra shear catalog # Extra shear catalog
# (currently not used) # (currently not used)
...@@ -163,6 +157,8 @@ shear_setting: ...@@ -163,6 +157,8 @@ shear_setting:
############################################### ###############################################
ins_effects: ins_effects:
# switches # switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise add_dark: ON # Whether to add dark noise
...@@ -183,20 +179,20 @@ ins_effects: ...@@ -183,20 +179,20 @@ ins_effects:
add_deadpixels: ON # Whether to add dead(dark) pixels add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# values # Values:
dark_exptime: 300 # Exposure time for dark current frames [seconds] # default values have been defined individually for each chip in:
flat_exptime: 150 # Exposure time for flat-fielding frames [seconds] # ObservationSim/Instrument/data/ccd/chip_definition.json
readout_time: 40 # The read-out time for each channel [seconds] # Set them here will override the default values
df_strength: 2.3 # Sillicon sensor diffusion strength # dark_exptime: 300 # Exposure time for dark current frames [seconds]
bias_level: 500 # bias level [e-/pixel] # flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
gain: 1.1 # Gain # readout_time: 40 # The read-out time for each channel [seconds]
full_well: 90000 # Full well depth [e-] # df_strength: 2.3 # Sillicon sensor diffusion strength
NBias: 1 # Number of bias frames to be exported for each exposure # bias_level: 500 # bias level [e-/pixel]
NDark: 1 # Number of dark frames to be exported for each exposure # gain: 1.1 # Gain
NFlat: 1 # Number of flat frames to be exported for each exposure # full_well: 90000 # Full well depth [e-]
############################################### ###############################################
# Output options # Output options (for calibration pointings only)
############################################### ###############################################
output_setting: output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
...@@ -205,12 +201,14 @@ output_setting: ...@@ -205,12 +201,14 @@ output_setting:
dark_output: ON # Whether to export the combined dark current files dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
############################################### ###############################################
# Random seeds # Random seeds
############################################### ###############################################
random_seeds: random_seeds:
seed_Av: 121212 # Seed for generating random intrinsic extinction
seed_poisson: 20210601 # Seed for Poisson noise seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields seed_flat: 20210101 # Seed for generating random flat fields
......
config/config_C6_fits.yaml 0 → 100644
+ 228
- 0
View file @ fd6c3108
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/share/simudata/CSSOSDataProductsSims/data/CSSTSimImage_C6/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "C6_fits_testRun"
# Whether to use MPI
run_option:
use_mpi: YES
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_C6_20221212"
star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
galaxy_cat: "cat2CSSTSim_bundle/"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
stamp_cat: "stampCatsIndex.hdf5"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Catalog_C6_20221212/sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
#stamp_SED:
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
# With stamp?
stamp_yes: YES
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 244.972773
dec_center: 39.895901
# Image rotation [degree]
image_rot: 109.59
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_radec_246.5_40.dat"
# pointing_dir: null
# pointing_file: null
# Number of calibration pointings
np_cal: 0
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [7]
# Whether to enable astrometric modeling
enable_astrometric_model: False
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog
# "extra": from seprate file
shear_type: "catalog"
# For constant shear filed
reduced_g1: 0.
reduced_g2: 0.
# Extra shear catalog
# (currently not used)
# shear_cat: "mockShear.cat"
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: OFF # Whether to add field distortions
add_back: OFF # Whether to add sky background
add_dark: OFF # Whether to add dark noise
add_readout: OFF # Whether to add read-out (Gaussian) noise
add_bias: OFF # Whether to add bias-level to images
bias_16channel: OFF # Whether to add different biases for 16 channels
gain_16channel: OFF # Whether to make different gains for 16 channels
shutter_effect: OFF # Whether to add shutter effect
flat_fielding: OFF # Whether to add flat-fielding effect
prnu_effect: OFF # Whether to add PRNU effect
non_linear: OFF # Whether to add non-linearity
cosmic_ray: OFF # Whether to add cosmic-ray
cray_differ: OFF # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: OFF # Whether to simulate CTE trails
saturbloom: OFF # Whether to simulate Saturation & Blooming
add_badcolumns: OFF # Whether to add bad columns
add_hotpixels: OFF # Whether to add hot pixels
add_deadpixels: OFF # Whether to add dead(dark) pixels
bright_fatter: OFF # Whether to simulate Brighter-Fatter (also diffusion) effect
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 0.01 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 2000 # bias level [e-/pixel]
# gain: 1. # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
config/config_NGP.yaml
+ 79
- 76
View file @ fd6c3108
...@@ -2,24 +2,16 @@ ...@@ -2,24 +2,16 @@
############################################### ###############################################
# #
# Configuration file for CSST simulation # Configuration file for CSST simulation
# CSST-Sim Group, 2021/10/07 # CSST-Sim Group, 2023/04/25
# #
############################################### ###############################################
# Base diretories and naming setup # Base diretories and naming setup
# Can add some of the command-line arguments here as well; # Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent # OK to pass either way or both, as long as they are consistent
# work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/" work_dir: "/share/home/fangyuedong/test_psf_rot/csst-simulation/workplace/"
work_dir: "/public/home/fangyuedong/test/CSST/workplace/" data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
data_dir: "/data/simudata/CSSOSDataProductsSims/data/" run_name: "NGP_C3"
run_name: "TEST_Speed"
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/data/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_test_NGP_2.17.dat"
# Whether to use MPI # Whether to use MPI
run_option: run_option:
...@@ -33,12 +25,32 @@ run_option: ...@@ -33,12 +25,32 @@ run_option:
# If yes, no imaging simulation will run # If yes, no imaging simulation will run
out_cat_only: NO out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "OnOrbitCalibration/CTargets20211231"
star_cat: "CT-NGP_r1.8_G28.hdf5"
galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Templates/Galaxy/"
# Only simulate stars? # Only simulate stars?
star_only: NO star_only: NO
# Only simulate galaxies? # Only simulate galaxies?
galaxy_only: NO galaxy_only: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
############################################### ###############################################
# Observation setting # Observation setting
############################################### ###############################################
...@@ -47,14 +59,14 @@ obs_setting: ...@@ -47,14 +59,14 @@ obs_setting:
# Options for survey types: # Options for survey types:
# "Photometric": simulate photometric chips only # "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only # "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane # "All": simulate full focal plane
survey_type: "All" survey_type: "Photometric"
# Exposure time [seconds] # Exposure time [seconds]
exp_time: 150. exp_time: 150.
# Observation starting date & time # Observation starting date & time
# (Subject to change)
date_obs: "210525" # [yymmdd] date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss] time_obs: "120000" # [hhmmss]
...@@ -62,10 +74,16 @@ obs_setting: ...@@ -62,10 +74,16 @@ obs_setting:
# Note: NOT valid when a pointing list file is specified # Note: NOT valid when a pointing list file is specified
ra_center: 192.8595 ra_center: 192.8595
dec_center: 27.1283 dec_center: 27.1283
# Image rotation [degree] # Image rotation [degree]
image_rot: -113.4333 image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_test_NGP_2.17.dat"
# Number of calibration pointings # Number of calibration pointings
np_cal: 0 np_cal: 0
...@@ -73,34 +91,25 @@ obs_setting: ...@@ -73,34 +91,25 @@ obs_setting:
# - give a list of indexes of pointings: [ip_1, ip_2...] # - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null # - run all pointings: null
# Note: only valid when a pointing list is specified # Note: only valid when a pointing list is specified
run_pointings: [0] run_pointings: [0, 1, 2]
# Run specific chip(s): # Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...] # - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null # - run all chips: null
# Note: for all pointings # Note: for all pointings
run_chips: [24] run_chips: null
# Whether to enable astrometric modeling # Whether to enable astrometric modeling
# astrometric_lib: "libshao.so"
enable_astrometric_model: True enable_astrometric_model: True
# Cut by saturation/limiting magnitude in which band? # Cut by saturation magnitude in which band?
cut_in_band: "g" cut_in_band: "z"
############################################### # saturation magnitude margin
# Input path setting mag_sat_margin: -2.5
###############################################
# Default path settings for WIDE survey simulation
input_path:
cat_dir: "OnOrbitCalibration/CTargets20211231"
star_cat: "CT-NGP_r1.8_G28.hdf5"
galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5"
SED_templates_path: # limiting magnitude margin
star_SED: "Catalog_20210126/SpecLib.hdf5" mag_lim_margin: +1.0
galaxy_SED: "Templates/Galaxy/"
############################################### ###############################################
# PSF setting # PSF setting
...@@ -119,13 +128,7 @@ psf_setting: ...@@ -119,13 +128,7 @@ psf_setting:
# path to PSF data # path to PSF data
# NOTE: only valid for "Interp" PSF # NOTE: only valid for "Interp" PSF
psf_dir: "/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube" psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube"
# path to field-distortion model
# Note: only valid when ins_effects: field_dist is "ON"
fd_path: "FieldDistModelGlobal_v1.0.pickle"
# sigma_spin: 0.0 # psf spin?
############################################### ###############################################
# Shear setting # Shear setting
...@@ -134,7 +137,7 @@ psf_setting: ...@@ -134,7 +137,7 @@ psf_setting:
shear_setting: shear_setting:
# Options to generate mock shear field: # Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear # "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog (not available yet) # "catalog": from catalog
# "extra": from seprate file # "extra": from seprate file
shear_type: "constant" shear_type: "constant"
...@@ -142,12 +145,6 @@ shear_setting: ...@@ -142,12 +145,6 @@ shear_setting:
reduced_g1: 0.026 reduced_g1: 0.026
reduced_g2: 0.015 reduced_g2: 0.015
# Representation of the shear vector?
reShear: "E"
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
# Extra shear catalog # Extra shear catalog
# (currently not used) # (currently not used)
# shear_cat: "mockShear.cat" # shear_cat: "mockShear.cat"
...@@ -157,38 +154,42 @@ shear_setting: ...@@ -157,38 +154,42 @@ shear_setting:
############################################### ###############################################
ins_effects: ins_effects:
# switches # switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise add_dark: ON # Whether to add dark noise
add_readout: ON # Whether to add read-out (Gaussian) noise add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images add_bias: ON # Whether to add bias-level to images
shutter_effect: ON # Whether to add shutter effect bias_16channel: ON # Whether to add different biases for 16 channels
flat_fielding: ON # Whether to add flat-fielding effect gain_16channel: ON # Whether to make different gains for 16 channels
prnu_effect: ON # Whether to add PRNU effect shutter_effect: ON # Whether to add shutter effect
non_linear: ON # Whether to add non-linearity flat_fielding: ON # Whether to add flat-fielding effect
cosmic_ray: ON # Whether to add cosmic-ray prnu_effect: ON # Whether to add PRNU effect
cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output non_linear: ON # Whether to add non-linearity
cte_trail: ON # Whether to simulate CTE trails cosmic_ray: ON # Whether to add cosmic-ray
saturbloom: ON # Whether to simulate Saturation & Blooming cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output
add_badcolumns: ON # Whether to add bad columns cte_trail: ON # Whether to simulate CTE trails
add_hotpixels: ON # Whether to add hot pixels saturbloom: ON # Whether to simulate Saturation & Blooming
add_deadpixels: ON # Whether to add dead(dark) pixels add_badcolumns: ON # Whether to add bad columns
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect add_hotpixels: ON # Whether to add hot pixels
add_deadpixels: ON # Whether to add dead(dark) pixels
# values bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
dark_exptime: 300 # Exposure time for dark current frames [seconds]
flat_exptime: 150 # Exposure time for flat-fielding frames [seconds] # Values:
readout_time: 40 # The read-out time for each channel [seconds] # default values have been defined individually for each chip in:
df_strength: 2.3 # Sillicon sensor diffusion strength # ObservationSim/Instrument/data/ccd/chip_definition.json
bias_level: 500 # bias level [e-/pixel] # Set them here will override the default values
gain: 1.1 # Gain # dark_exptime: 300 # Exposure time for dark current frames [seconds]
full_well: 90000 # Full well depth [e-] # flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
NBias: 1 # Number of bias frames to be exported for each exposure # readout_time: 40 # The read-out time for each channel [seconds]
NDark: 1 # Number of dark frames to be exported for each exposure # df_strength: 2.3 # Sillicon sensor diffusion strength
NFlat: 1 # Number of flat frames to be exported for each exposure # bias_level: 500 # bias level [e-/pixel]
# gain: 1.1 # Gain
############################################### # full_well: 90000 # Full well depth [e-]
# Output options
###############################################
# Output options (for calibration pointings only)
############################################### ###############################################
output_setting: output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
...@@ -197,12 +198,14 @@ output_setting: ...@@ -197,12 +198,14 @@ output_setting:
dark_output: ON # Whether to export the combined dark current files dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
############################################### ###############################################
# Random seeds # Random seeds
############################################### ###############################################
random_seeds: random_seeds:
seed_Av: 121212 # Seed for generating random intrinsic extinction
seed_poisson: 20210601 # Seed for Poisson noise seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields seed_flat: 20210101 # Seed for generating random flat fields
......
config/config_example.yaml
+ 59
- 58
View file @ fd6c3108
...@@ -2,24 +2,17 @@ ...@@ -2,24 +2,17 @@
############################################### ###############################################
# #
# Configuration file for CSST simulation # Configuration file for CSST simulation
# CSST-Sim Group, 2021/10/07 # CSST-Sim Group, 2023/04/25
# #
############################################### ###############################################
# Base diretories and naming setup # Base diretories and naming setup
# Can add some of the command-line arguments here as well; # Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent # OK to pass either way or both, as long as they are consistent
# work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/"
work_dir: "/public/home/fangyuedong/20211203/CSST/workplace/" work_dir: "/public/home/fangyuedong/20211203/CSST/workplace/"
data_dir: "/data/simudata/CSSOSDataProductsSims/data/" data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
run_name: "example_20211217" run_name: "example"
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: null
pointing_file: null
# Whether to use MPI # Whether to use MPI
run_option: run_option:
...@@ -27,16 +20,29 @@ run_option: ...@@ -27,16 +20,29 @@ run_option:
# NOTE: "n_threads" paramters is currently not used in the backend # NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend # simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster # in order to config the number of threads to request from NAOC cluster
n_threads: 40 n_threads: 80
# Output catalog only? # Output catalog only?
# If yes, no imaging simulation will run # If yes, no imaging simulation will run
out_cat_only: NO out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "catalog_points_7degree2/point_RA60.9624_DE-41.5032/"
star_cat: "stars_ccd13_p_RA60.9624_DE-41.5032.hdf5"
galaxy_cat: null
SED_templates_path:
star_SED: "SED_MMW_Gaia_Cluster_D20_SS.hdf5"
galaxy_SED: null
# Only simulate stars? # Only simulate stars?
star_only: NO star_only: YES
# Only simulate galaxies?
galaxy_only: NO
############################################### ###############################################
# Observation setting # Observation setting
...@@ -46,14 +52,14 @@ obs_setting: ...@@ -46,14 +52,14 @@ obs_setting:
# Options for survey types: # Options for survey types:
# "Photometric": simulate photometric chips only # "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only # "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane # "All": simulate full focal plane
survey_type: "All" survey_type: "Photometric"
# Exposure time [seconds] # Exposure time [seconds]
exp_time: 150. exp_time: 150.
# Observation starting date & time # Observation starting date & time
# (Subject to change)
date_obs: "210525" # [yymmdd] date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss] time_obs: "120000" # [hhmmss]
...@@ -61,10 +67,16 @@ obs_setting: ...@@ -61,10 +67,16 @@ obs_setting:
# Note: NOT valid when a pointing list file is specified # Note: NOT valid when a pointing list file is specified
ra_center: 60.9624 ra_center: 60.9624
dec_center: -41.5032 dec_center: -41.5032
# Image rotation [degree] # Image rotation [degree]
image_rot: -113.4333 image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: null
pointing_file: null
# Number of calibration pointings # Number of calibration pointings
np_cal: 2 np_cal: 2
...@@ -80,24 +92,19 @@ obs_setting: ...@@ -80,24 +92,19 @@ obs_setting:
# Note: for all pointings # Note: for all pointings
run_chips: [18] run_chips: [18]
# Cut by saturation/limiting magnitude in which band? # Whether to enable astrometric modeling
cut_in_band: "g" enable_astrometric_model: True
############################################### # Cut by saturation magnitude in which band?
# Input path setting cut_in_band: "z"
###############################################
# Default path settings for WIDE survey simulation # saturation magnitude margin
input_path: mag_sat_margin: -2.5
cat_dir: "catalog_points_7degree2/point_RA60.9624_DE-41.5032/"
star_cat: "stars_ccd13_p_RA60.9624_DE-41.5032.hdf5"
galaxy_cat: null
SED_templates_path: # limiting magnitude margin
star_SED: "SED_MMW_Gaia_Cluster_D20_SS.hdf5" mag_lim_margin: +1.0
galaxy_SED: null
################################################ ###############################################
# PSF setting # PSF setting
############################################### ###############################################
psf_setting: psf_setting:
...@@ -116,12 +123,6 @@ psf_setting: ...@@ -116,12 +123,6 @@ psf_setting:
# NOTE: only valid for "Interp" PSF # NOTE: only valid for "Interp" PSF
psf_dir: "/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube" psf_dir: "/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube"
# path to field-distortion model
# Note: only valid when ins_effects: field_dist is "ON"
fd_path: "FieldDistModelGlobal_v1.0.pickle"
# sigma_spin: 0.0 # psf spin?
############################################### ###############################################
# Shear setting # Shear setting
############################################### ###############################################
...@@ -129,7 +130,7 @@ psf_setting: ...@@ -129,7 +130,7 @@ psf_setting:
shear_setting: shear_setting:
# Options to generate mock shear field: # Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear # "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog (not available yet) # "catalog": from catalog
# "extra": from seprate file # "extra": from seprate file
shear_type: "constant" shear_type: "constant"
...@@ -137,12 +138,6 @@ shear_setting: ...@@ -137,12 +138,6 @@ shear_setting:
reduced_g1: 0.026 reduced_g1: 0.026
reduced_g2: 0.015 reduced_g2: 0.015
# Representation of the shear vector?
reShear: "E"
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
# Extra shear catalog # Extra shear catalog
# (currently not used) # (currently not used)
# shear_cat: "mockShear.cat" # shear_cat: "mockShear.cat"
...@@ -152,11 +147,15 @@ shear_setting: ...@@ -152,11 +147,15 @@ shear_setting:
############################################### ###############################################
ins_effects: ins_effects:
# switches # switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise add_dark: ON # Whether to add dark noise
add_readout: ON # Whether to add read-out (Gaussian) noise add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images add_bias: ON # Whether to add bias-level to images
bias_16channel: ON # Whether to add different biases for 16 channels
gain_16channel: ON # Whether to make different gains for 16 channels
shutter_effect: ON # Whether to add shutter effect shutter_effect: ON # Whether to add shutter effect
flat_fielding: ON # Whether to add flat-fielding effect flat_fielding: ON # Whether to add flat-fielding effect
prnu_effect: ON # Whether to add PRNU effect prnu_effect: ON # Whether to add PRNU effect
...@@ -170,20 +169,20 @@ ins_effects: ...@@ -170,20 +169,20 @@ ins_effects:
add_deadpixels: ON # Whether to add dead(dark) pixels add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# values # Values:
dark_exptime: 300 # Exposure time for dark current frames [seconds] # default values have been defined individually for each chip in:
flat_exptime: 150 # Exposure time for flat-fielding frames [seconds] # ObservationSim/Instrument/data/ccd/chip_definition.json
readout_time: 40 # The read-out time for each channel [seconds] # Set them here will override the default values
df_strength: 2.3 # Sillicon sensor diffusion strength # dark_exptime: 300 # Exposure time for dark current frames [seconds]
bias_level: 500 # bias level [e-/pixel] # flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
gain: 1.1 # Gain # readout_time: 40 # The read-out time for each channel [seconds]
full_well: 90000 # Full well depth [e-] # df_strength: 2.3 # Sillicon sensor diffusion strength
NBias: 1 # Number of bias frames to be exported for each exposure # bias_level: 500 # bias level [e-/pixel]
NDark: 1 # Number of dark frames to be exported for each exposure # gain: 1.1 # Gain
NFlat: 1 # Number of flat frames to be exported for each exposure # full_well: 90000 # Full well depth [e-]
############################################### ###############################################
# Output options # Output options (for calibration pointings only)
############################################### ###############################################
output_setting: output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
...@@ -192,12 +191,14 @@ output_setting: ...@@ -192,12 +191,14 @@ output_setting:
dark_output: ON # Whether to export the combined dark current files dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
############################################### ###############################################
# Random seeds # Random seeds
############################################### ###############################################
random_seeds: random_seeds:
seed_Av: 121212 # Seed for generating random intrinsic extinction
seed_poisson: 20210601 # Seed for Poisson noise seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields seed_flat: 20210101 # Seed for generating random flat fields
......
config/config_C3.yaml → config/config_fgs.yaml
+ 75
- 66
View file @ fd6c3108
...@@ -2,23 +2,16 @@ ...@@ -2,23 +2,16 @@
############################################### ###############################################
# #
# Configuration file for CSST simulation # Configuration file for CSST simulation
# CSST-Sim Group, 2021/10/07 # CSST-Sim Group, 2023/04/25
# #
############################################### ###############################################
# Base diretories and naming setup # Base diretories and naming setup
# Can add some of the command-line arguments here as well; # Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent # OK to pass either way or both, as long as they are consistent
work_dir: "/public/home/fangyuedong/temp/CSST/workplace/" work_dir: "/share/home/fangyuedong/fgs_sim/csst-simulation/workplace/"
data_dir: "/data/simudata/CSSOSDataProductsSims/data/" data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "TEST_random_knots" run_name: "fgs_test"
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/data/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_test1214.dat"
# Whether to use MPI # Whether to use MPI
run_option: run_option:
...@@ -32,12 +25,35 @@ run_option: ...@@ -32,12 +25,35 @@ run_option:
# If yes, no imaging simulation will run # If yes, no imaging simulation will run
out_cat_only: NO out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_C6_20221212"
star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
galaxy_cat: "cat2CSSTSim_bundle/"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Catalog_C6_20221212/sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
# Only simulate stars? # Only simulate stars?
star_only: NO star_only: NO
# Only simulate galaxies? # Only simulate galaxies?
galaxy_only: NO galaxy_only: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
############################################### ###############################################
# Observation setting # Observation setting
############################################### ###############################################
...@@ -46,24 +62,32 @@ obs_setting: ...@@ -46,24 +62,32 @@ obs_setting:
# Options for survey types: # Options for survey types:
# "Photometric": simulate photometric chips only # "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only # "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane # "All": simulate full focal plane
survey_type: "All" survey_type: "FGS"
# Exposure time [seconds] # Exposure time [seconds]
exp_time: 150. exp_time: 150.
# Observation starting date & time # Observation starting date & time
# (Subject to change)
date_obs: "210525" # [yymmdd] date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss] time_obs: "120000" # [hhmmss]
# Default Pointing [degrees] # Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified # Note: NOT valid when a pointing list file is specified
ra_center: 60. ra_center: 244.972773
dec_center: -40. dec_center: 39.895901
# Image rotation [degree] # Image rotation [degree]
image_rot: -113.4333 image_rot: 109.59
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
# pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
# pointing_file: "pointing_radec_246.5_40.dat"
pointing_dir: null
pointing_file: null
# Number of calibration pointings # Number of calibration pointings
np_cal: 0 np_cal: 0
...@@ -72,34 +96,25 @@ obs_setting: ...@@ -72,34 +96,25 @@ obs_setting:
# - give a list of indexes of pointings: [ip_1, ip_2...] # - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null # - run all pointings: null
# Note: only valid when a pointing list is specified # Note: only valid when a pointing list is specified
run_pointings: [1] run_pointings: [0]
# Run specific chip(s): # Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...] # - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null # - run all chips: null
# Note: for all pointings # Note: for all pointings
run_chips: [1, 26, 29, 6, 7, 22, 23, 19, 20] run_chips: [32]
# Whether to enable astrometric modeling # Whether to enable astrometric modeling
# astrometric_lib: "libshao.so"
enable_astrometric_model: True enable_astrometric_model: True
# Cut by saturation/limiting magnitude in which band? # Cut by saturation magnitude in which band?
cut_in_band: "g" cut_in_band: "FGS"
###############################################
# Input path setting
###############################################
# Default path settings for WIDE survey simulation # saturation magnitude margin
input_path: mag_sat_margin: -2.5
cat_dir: "Catalog_20210126/"
star_cat: "MMW_Cluster_D20_healpix_21.hdf5"
galaxy_cat: "galaxyCats_r_10.0_healpix.hdf5"
SED_templates_path: # limiting magnitude margin
star_SED: "Catalog_20210126/SpecLib.hdf5" mag_lim_margin: +1.0
galaxy_SED: "Templates/Galaxy/"
############################################### ###############################################
# PSF setting # PSF setting
...@@ -118,13 +133,7 @@ psf_setting: ...@@ -118,13 +133,7 @@ psf_setting:
# path to PSF data # path to PSF data
# NOTE: only valid for "Interp" PSF # NOTE: only valid for "Interp" PSF
psf_dir: "/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube" psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
# path to field-distortion model
# Note: only valid when ins_effects: field_dist is "ON"
fd_path: "/data/simudata/CSSOSDataProductsSims/data/FieldDistModelGlobal_v1.0.pickle"
# sigma_spin: 0.0 # psf spin?
############################################### ###############################################
# Shear setting # Shear setting
...@@ -133,19 +142,13 @@ psf_setting: ...@@ -133,19 +142,13 @@ psf_setting:
shear_setting: shear_setting:
# Options to generate mock shear field: # Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear # "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog (not available yet) # "catalog": from catalog
# "extra": from seprate file # "extra": from seprate file
shear_type: "constant" shear_type: "catalog"
# For constant shear filed # For constant shear filed
reduced_g1: 0.026 reduced_g1: 0.
reduced_g2: 0.015 reduced_g2: 0.
# Representation of the shear vector?
reShear: "E"
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
# Extra shear catalog # Extra shear catalog
# (currently not used) # (currently not used)
...@@ -156,15 +159,19 @@ shear_setting: ...@@ -156,15 +159,19 @@ shear_setting:
############################################### ###############################################
ins_effects: ins_effects:
# switches # switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise add_dark: ON # Whether to add dark noise
add_readout: ON # Whether to add read-out (Gaussian) noise add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images add_bias: ON # Whether to add bias-level to images
shutter_effect: ON # Whether to add shutter effect bias_16channel: OFF # Whether to add different biases for 16 channels
gain_16channel: OFF # Whether to make different gains for 16 channels
shutter_effect: OFF # Whether to add shutter effect
flat_fielding: ON # Whether to add flat-fielding effect flat_fielding: ON # Whether to add flat-fielding effect
prnu_effect: ON # Whether to add PRNU effect prnu_effect: ON # Whether to add PRNU effect
non_linear: OFF # Whether to add non-linearity non_linear: ON # Whether to add non-linearity
cosmic_ray: ON # Whether to add cosmic-ray cosmic_ray: ON # Whether to add cosmic-ray
cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: ON # Whether to simulate CTE trails cte_trail: ON # Whether to simulate CTE trails
...@@ -174,20 +181,20 @@ ins_effects: ...@@ -174,20 +181,20 @@ ins_effects:
add_deadpixels: ON # Whether to add dead(dark) pixels add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# values # Values:
dark_exptime: 300 # Exposure time for dark current frames [seconds] # default values have been defined individually for each chip in:
flat_exptime: 150 # Exposure time for flat-fielding frames [seconds] # ObservationSim/Instrument/data/ccd/chip_definition.json
readout_time: 40 # The read-out time for each channel [seconds] # Set them here will override the default values
df_strength: 2.3 # Sillicon sensor diffusion strength # dark_exptime: 300 # Exposure time for dark current frames [seconds]
bias_level: 500 # bias level [e-/pixel] # flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
gain: 1.1 # Gain # readout_time: 0.01 # The read-out time for each channel [seconds]
full_well: 90000 # Full well depth [e-] # df_strength: 2.3 # Sillicon sensor diffusion strength
NBias: 1 # Number of bias frames to be exported for each exposure # bias_level: 2000 # bias level [e-/pixel]
NDark: 1 # Number of dark frames to be exported for each exposure # gain: 1. # Gain
NFlat: 1 # Number of flat frames to be exported for each exposure # full_well: 90000 # Full well depth [e-]
############################################### ###############################################
# Output options # Output options (for calibration pointings only)
############################################### ###############################################
output_setting: output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
...@@ -196,12 +203,14 @@ output_setting: ...@@ -196,12 +203,14 @@ output_setting:
dark_output: ON # Whether to export the combined dark current files dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
############################################### ###############################################
# Random seeds # Random seeds
############################################### ###############################################
random_seeds: random_seeds:
seed_Av: 121212 # Seed for generating random intrinsic extinction
seed_poisson: 20210601 # Seed for Poisson noise seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields seed_flat: 20210101 # Seed for generating random flat fields
......
config/config_testCASE.yaml 0 → 100644
+ 227
- 0
View file @ fd6c3108
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2023/04/30
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/share/simudata/CSSOSDataProductsSims/data/CSSTSimImage_C6/"
data_dir: "/share/home/weichengliang/CSST_git/csst-simulation/tools/"
run_name: "C6_fits_testRun"
# Whether to use MPI
run_option:
use_mpi: YES
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_test"
star_cat: "testStarCat.h5"
galaxy_cat: "testGalaxyCat.h5"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
stamp_cat: "stampCatsIndex.hdf5"
SED_templates_path:
star_SED: "SpecLib.hdf5"
galaxy_SED: "sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
#stamp_SED:
# simulate stars?
star_yes: NO
# simulate galaxies?
galaxy_yes: NO
# With stamp?
stamp_yes: YES
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 60.11828178499743 #244.972773
dec_center: -39.75897455697294 #39.895901
# Image rotation [degree]
image_rot: 112.23685624012192 #109.59
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: null #"/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: null #"pointing_radec_246.5_40.dat"
# Number of calibration pointings
np_cal: 0
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [7, 8, 9]
# Whether to enable astrometric modeling
enable_astrometric_model: False
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog
# "extra": from seprate file
shear_type: "catalog"
# For constant shear filed
reduced_g1: 0.
reduced_g2: 0.
# Extra shear catalog
# (currently not used)
# shear_cat: "mockShear.cat"
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions
add_back: OFF # Whether to add sky background
add_dark: OFF # Whether to add dark noise
add_readout: OFF # Whether to add read-out (Gaussian) noise
add_bias: OFF # Whether to add bias-level to images
bias_16channel: OFF # Whether to add different biases for 16 channels
gain_16channel: OFF # Whether to make different gains for 16 channels
shutter_effect: OFF # Whether to add shutter effect
flat_fielding: OFF # Whether to add flat-fielding effect
prnu_effect: OFF # Whether to add PRNU effect
non_linear: OFF # Whether to add non-linearity
cosmic_ray: OFF # Whether to add cosmic-ray
cray_differ: OFF # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: OFF # Whether to simulate CTE trails
saturbloom: OFF # Whether to simulate Saturation & Blooming
add_badcolumns: OFF # Whether to add bad columns
add_hotpixels: OFF # Whether to add hot pixels
add_deadpixels: OFF # Whether to add dead(dark) pixels
bright_fatter: OFF # Whether to simulate Brighter-Fatter (also diffusion) effect
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 0.01 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 2000 # bias level [e-/pixel]
# gain: 1. # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
example.sh
+ 4
- 1
View file @ fd6c3108
...@@ -2,4 +2,7 @@ ...@@ -2,4 +2,7 @@
date date
python /public/home/fangyuedong/temp/CSST/run_example.py config_example.yaml -c /public/home/fangyuedong/temp/CSST/config python3 /share/home/fangyuedong/fgs_sim/csst-simulation/run_sim.py \
--config_file config_example.yaml \
--catalog Catalog_example \
-c /share/home/fangyuedong/fgs_sim/csst-simulation/config
mpi_example.pbs
+ 4
- 1
View file @ fd6c3108
...@@ -16,4 +16,7 @@ NP=40 ...@@ -16,4 +16,7 @@ NP=40
date date
echo $NP echo $NP
mpirun -np $NP python /public/home/fangyuedong/temp/CSST/run_sim.py config_example.yaml -c /public/home/fangyuedong/temp/CSST/config mpirun -np $NP python /share/home/fangyuedong/fgs_sim/csst-simulation/run_sim.py \
--config_file config_example.yaml \
--catalog Catalog_example \
-c /share/home/fangyuedong/fgs_sim/csst-simulation/config
\ No newline at end of file
run_C3.pbs deleted 100755 → 0
+ 0
- 19
View file @ ca7fb5db
#!/bin/bash
#PBS -N SIMS
##PBS -l walltime=70:00:00
##mpdallexit
##mpdboot -n 10 -f ./mpd.hosts
##PBS -j oe
#PBS -l nodes=comput108:ppn=80
#####PBS -q longq
#PBS -q batch
#PBS -u fangyuedong
NP=80
date
echo $NP
mpirun -np $NP python /public/home/fangyuedong/temp/CSST/run_sim.py config_C3.yaml -c /public/home/fangyuedong/temp/CSST/config
run_C6.pbs 0 → 100755
+ 18
- 0
View file @ fd6c3108
#!/bin/bash
#PBS -N SIMS
#PBS -l nodes=wcl-1:ppn=60
###PBS -l nodes=wcl-1:ppn=24+wcl-2:ppn=24+wcl-3:ppn=24+wcl-4:ppn=24+wcl-5:ppn=24+wcl-6:ppn=24
#PBS -u fangyuedong
###PBS -j oe
cd $PBS_O_WORKDIR
NP=60
date
mpirun -np $NP python3 /share/home/fangyuedong/fgs_sim/csst-simulation/run_sim.py \
--config_file config_C6.yaml \
--catalog C6_Catalog \
-c /share/home/fangyuedong/fgs_sim/csst-simulation/config
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment