Commit fd6c3108 authored by Fang Yuedong's avatar Fang Yuedong Committed by Zhang Xin
Browse files

Release v2.0

parent ca7fb5db
......@@ -13,6 +13,11 @@ class Quasar(MockObject):
super().__init__(param, logger=logger)
def load_SED(self, survey_type, sed_path=None, cosids=None, objtypes=None, sed_templates=None, normFilter=None, target_filt=None):
'''
--------------------------------------------------------------------
(Deprecated) Left over codes from cycle-1 to cycle-3 implementations
--------------------------------------------------------------------
'''
if survey_type == "photometric":
norm_thr_rang_ids = normFilter['SENSITIVITY'] > 0.001
if sed_templates is None:
......
......@@ -185,8 +185,8 @@ class SpecDisperser(object):
origin_in[1] = self.origin[1]
dx0_in = dx[0]
dy0_in = dyc[0]
originOut_x = origin_in[1] + dx0_in - 1
originOut_y = origin_in[0] + dy0_in - 1
originOut_x = origin_in[1] + dx0_in
originOut_y = origin_in[0] + dy0_in
if self.flat_cube is None:
beam_flat = None
......@@ -248,7 +248,7 @@ class SpecDisperser(object):
if self.isAlongY == 1:
model, _, _ = rotate90(array_orig=model, isClockwise=0)
return model, originOut_x, originOut_y
return model, originOut_x, originOut_y, dxpix, dypix, lam_beam,ysens
def writerSensitivityFile(self, conffile='', beam='', w=None, sens=None):
orders = {'A': '1st', 'B': '0st', 'C': '2st', 'D': '-1st', 'E': '-2st'}
......
import os, sys
import random
import numpy as np
import astropy.constants as cons
from astropy.table import Table
from scipy import interpolate
import astropy.io.fits as fitsio
import galsim
import gc
from ObservationSim.MockObject.MockObject import MockObject
from ObservationSim.MockObject._util import magToFlux,VC_A
from ObservationSim.MockObject._util import eObs, integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG,convolveGaussXorders
class Stamp(MockObject):
def __init__(self, param):
super().__init__(param)
def unload_SED(self):
"""(Test) free up SED memory
"""
del self.sed
def drawObj_multiband(self, tel, pos_img, psf_model, bandpass_list, filt, chip, nphotons_tot=None, g1=0, g2=0, exptime=150., fd_shear=None):
if nphotons_tot == None:
nphotons_tot = self.getElectronFluxFilt(filt, tel, exptime)
try:
full = integrate_sed_bandpass(sed=self.sed, bandpass=filt.bandpass_full)
except Exception as e:
print(e)
self.logger.error(e)
return False
nphotons_sum = 0
photons_list = []
xmax, ymax = 0, 0
if self.getMagFilter(filt) <= 15:
folding_threshold = 5.e-4
else:
folding_threshold = 5.e-3
gsp = galsim.GSParams(folding_threshold=folding_threshold)
self.real_pos = self.getRealPos(chip.img, global_x=self.posImg.x, global_y=self.posImg.y,
img_real_wcs=self.real_wcs)
x, y = self.real_pos.x + 0.5, self.real_pos.y + 0.5
x_nominal = int(np.floor(x + 0.5))
y_nominal = int(np.floor(y + 0.5))
dx = x - x_nominal
dy = y - y_nominal
offset = galsim.PositionD(dx, dy)
real_wcs_local = self.real_wcs.local(self.real_pos)
for i in range(len(bandpass_list)):
bandpass = bandpass_list[i]
try:
sub = integrate_sed_bandpass(sed=self.sed, bandpass=bandpass)
except Exception as e:
print(e)
self.logger.error(e)
# return False
continue
ratio = sub/full
if not (ratio == -1 or (ratio != ratio)):
nphotons = ratio * nphotons_tot
else:
# return False
continue
nphotons_sum += nphotons
psf, pos_shear = psf_model.get_PSF(chip=chip, pos_img=pos_img, bandpass=bandpass, folding_threshold=folding_threshold)
_gal = self.param['image']
galIm = galsim.ImageF(_gal, scale=self.param['pixScale'])
gal = galsim.InterpolatedImage(galIm)
gal = gal.withFlux(nphotons)
#gal_shear = galsim.Shear(g1=g1, g2=g2)
#gal = gal.shear(gal_shear)
gal = galsim.Convolve(psf, gal)
if fd_shear is not None:
gal = gal.shear(fd_shear)
stamp = gal.drawImage(wcs=real_wcs_local, method='phot', offset=self.offset, save_photons=True)
xmax = max(xmax, stamp.xmax - stamp.xmin)
ymax = max(ymax, stamp.ymax - stamp.ymin)
photons = stamp.photons
photons.x += x_nominal
photons.y += y_nominal
photons_list.append(photons)
del gal
# print('xmax = %d, ymax = %d '%(xmax, ymax))
stamp = galsim.ImageF(int(xmax*1.1), int(ymax*1.1))
stamp.wcs = real_wcs_local
stamp.setCenter(x_nominal, y_nominal)
bounds = stamp.bounds & galsim.BoundsI(0, chip.npix_x - 1, 0, chip.npix_y - 1)
if bounds.area() > 0:
chip.img.setOrigin(0, 0)
stamp[bounds] = chip.img[bounds]
for i in range(len(photons_list)):
if i == 0:
chip.sensor.accumulate(photons_list[i], stamp)
else:
chip.sensor.accumulate(photons_list[i], stamp, resume=True)
chip.img[bounds] = stamp[bounds]
chip.img.setOrigin(chip.bound.xmin, chip.bound.ymin)
del photons_list
del stamp
gc.collect()
return True, pos_shear
......@@ -3,5 +3,6 @@ from .Galaxy import Galaxy
from .CatalogBase import CatalogBase
from .Quasar import Quasar
from .Star import Star
from .Stamp import Stamp
from .SkybackgroundMap import *
# from .CosmicRay import CosmicRay
\ No newline at end of file
# from .CosmicRay import CosmicRay
import numpy as np
import os
from scipy.interpolate import InterpolatedUnivariateSpline, interp1d
from scipy import interpolate
from scipy import interpolate, integrate
from astropy.table import Table
import galsim
......@@ -9,6 +9,14 @@ VC_A = 2.99792458e+18 # speed of light: A/s
VC_M = 2.99792458e+8 # speed of light: m/s
H_PLANK = 6.626196e-27 # Plank constant: erg s
def comoving_dist(z, om_m=0.3111, om_L=0.6889, h=0.6766):
# Return comving distance in pc
H0 = h*100. # km / (s Mpc)
def dist_int(z):
return 1./np.sqrt(om_m*(1.+z)**3 + om_L)
res, err = integrate.quad(dist_int, 0., z)
return [res * (VC_M/1e3/H0) * 1e6, err * (VC_M/1e3/H0) * 1e6]
def magToFlux(mag):
"""
flux of a given AB magnitude
......@@ -447,7 +455,10 @@ def getObservedSED(sedCat, redshift=0.0, av=0.0, redden=0):
sw, sf = sedCat[:,0], sedCat[:,1]
# reddening
sf = reddening(sw, sf, av=av, model=redden)
sw, sf = sw*z, sf*(z**3)
# sw, sf = sw*z, sf*(z**3)
sw, sf = sw*z, sf/z
# sw, sf = sw*z, sf
# lyman forest correction
sf = lyman_forest(sw, sf, redshift)
isedObs = (sw.copy(), sf.copy())
......
......@@ -4,6 +4,7 @@ import mpi4py.MPI as MPI
import galsim
import logging
import psutil
import gc
from astropy.io import fits
from datetime import datetime
......@@ -30,15 +31,8 @@ class Observation(object):
self.all_filter = []
self.Catalog = Catalog
# if we want to apply field distortion?
if self.config["ins_effects"]["field_dist"] == True:
self.fd_model = FieldDistortion(fdModel_path=self.path_dict["fd_path"])
else:
self.fd_model = None
# Construct chips & filters:
nchips = self.focal_plane.nchip_x*self.focal_plane.nchip_y
for i in range(nchips):
for i in range(self.focal_plane.nchips):
chipID = i + 1
# Make Chip & Filter lists
......@@ -54,45 +48,32 @@ class Observation(object):
self.filter_list.append(filt)
self.all_filter.append(filt)
# Read catalog and shear(s)
self.g1_field, self.g2_field, self.nshear = get_shear_field(config=self.config)
def run_one_chip(self, chip, filt, pointing, chip_output, wcs_fp=None, psf_model=None, shear_cat_file=None, cat_dir=None, sed_dir=None):
# print(':::::::::::::::::::Current Pointing Information::::::::::::::::::')
# print("RA: %f, DEC; %f" % (pointing.ra, pointing.dec))
# print("Time: %s" % datetime.fromtimestamp(pointing.timestamp).isoformat())
# print("Exposure time: %f" % pointing.exp_time)
# print("Satellite Position (x, y, z): (%f, %f, %f)" % (pointing.sat_x, pointing.sat_y, pointing.sat_z))
# print("Satellite Velocity (x, y, z): (%f, %f, %f)" % (pointing.sat_vx, pointing.sat_vy, pointing.sat_vz))
# print("Position Angle: %f" % pointing.img_pa.deg)
# print('Chip : %d' % chip.chipID)
# print(':::::::::::::::::::::::::::END:::::::::::::::::::::::::::::::::::')
chip_output.logger.info(':::::::::::::::::::Current Pointing Information::::::::::::::::::')
chip_output.logger.info("RA: %f, DEC; %f" % (pointing.ra, pointing.dec))
chip_output.logger.info("Time: %s" % datetime.fromtimestamp(pointing.timestamp).isoformat())
chip_output.logger.info("Exposure time: %f" % pointing.exp_time)
chip_output.logger.info("Satellite Position (x, y, z): (%f, %f, %f)" % (pointing.sat_x, pointing.sat_y, pointing.sat_z))
chip_output.logger.info("Satellite Velocity (x, y, z): (%f, %f, %f)" % (pointing.sat_vx, pointing.sat_vy, pointing.sat_vz))
chip_output.logger.info("Position Angle: %f" % pointing.img_pa.deg)
chip_output.logger.info('Chip : %d' % chip.chipID)
chip_output.logger.info(':::::::::::::::::::::::::::END:::::::::::::::::::::::::::::::::::')
chip_output.Log_info(':::::::::::::::::::Current Pointing Information::::::::::::::::::')
chip_output.Log_info("RA: %f, DEC; %f" % (pointing.ra, pointing.dec))
chip_output.Log_info("Time: %s" % datetime.utcfromtimestamp(pointing.timestamp).isoformat())
chip_output.Log_info("Exposure time: %f" % pointing.exp_time)
chip_output.Log_info("Satellite Position (x, y, z): (%f, %f, %f)" % (pointing.sat_x, pointing.sat_y, pointing.sat_z))
chip_output.Log_info("Satellite Velocity (x, y, z): (%f, %f, %f)" % (pointing.sat_vx, pointing.sat_vy, pointing.sat_vz))
chip_output.Log_info("Position Angle: %f" % pointing.img_pa.deg)
chip_output.Log_info('Chip : %d' % chip.chipID)
chip_output.Log_info(':::::::::::::::::::::::::::END:::::::::::::::::::::::::::::::::::')
if self.config["psf_setting"]["psf_model"] == "Gauss":
psf_model = PSFGauss(chip=chip, psfRa=self.config["psf_setting"]["psf_rcont"])
elif self.config["psf_setting"]["psf_model"] == "Interp":
psf_model = PSFInterp(chip=chip, PSF_data_file=self.path_dict["psf_dir"])
psf_model = PSFInterp(chip=chip, npsf=chip.n_psf_samples, PSF_data_file=self.path_dict["psf_dir"])
else:
# print("unrecognized PSF model type!!", flush=True)
chip_output.logger.error("unrecognized PSF model type!!", flush=True)
chip_output.Log_error("unrecognized PSF model type!!", flush=True)
# Get (extra) shear fields
# Figure out shear fields
if shear_cat_file is not None:
self.g1_field, self.g2_field, self.nshear = get_shear_field(config=self.config, shear_cat_file=shear_cat_file)
# Apply astrometric simulation for pointing
if self.config["obs_setting"]["enable_astrometric_model"]:
dt = datetime.fromtimestamp(pointing.timestamp)
dt = datetime.utcfromtimestamp(pointing.timestamp)
date_str = dt.date().isoformat()
time_str = dt.time().isoformat()
ra_cen, dec_cen = on_orbit_obs_position(
......@@ -109,7 +90,7 @@ class Observation(object):
input_vx=pointing.sat_vx,
input_vy=pointing.sat_vy,
input_vz=pointing.sat_vz,
input_epoch="J2015.5",
input_epoch="J2000",
input_date_str=date_str,
input_time_str=time_str
)
......@@ -128,24 +109,19 @@ class Observation(object):
chip.img.wcs = wcs_fp
if chip.survey_type == "photometric":
sky_map = None
# elif chip.survey_type == "spectroscopic":
# sky_map = calculateSkyMap_split_g(xLen=chip.npix_x, yLen=chip.npix_y, blueLimit=filt.blue_limit, redLimit=filt.red_limit, skyfn=self.path_dict["sky_file"], conf=chip.sls_conf, pixelSize=chip.pix_scale, isAlongY=0)
elif chip.survey_type == "spectroscopic":
# chip.loadSLSFLATCUBE(flat_fn='flat_cube.fits')
flat_normal = np.ones_like(chip.img.array)
if self.config["ins_effects"]["flat_fielding"] == True:
# print("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding"%chip.chipID, flush=True)
# print(chip.img.bounds, flush=True)
chip_output.logger.info("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding"%chip.chipID)
chip_output.Log_info("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding"%chip.chipID)
msg = str(chip.img.bounds)
chip_output.logger.info(msg)
chip_output.Log_info(msg)
flat_img = Effects.MakeFlatSmooth(
chip.img.bounds,
int(self.config["random_seeds"]["seed_flat"]))
flat_normal = flat_normal * flat_img.array / np.mean(flat_img.array)
if self.config["ins_effects"]["shutter_effect"] == True:
# print("SLS flat preprocess,CHIP %d : Apply shutter effect"%chip.chipID, flush=True)
chip_output.logger.info("SLS flat preprocess,CHIP %d : Apply shutter effect"%chip.chipID)
chip_output.Log_info("SLS flat preprocess,CHIP %d : Apply shutter effect"%chip.chipID)
shuttimg = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735,
dt=1E-3) # shutter effect normalized image for this chip
flat_normal = flat_normal*shuttimg
......@@ -158,12 +134,11 @@ class Observation(object):
pixelSize=chip.pix_scale,
isAlongY=0,
flat_cube=chip.flat_cube)
# sky_map = np.ones([9216, 9232])
del flat_normal
if pointing.pointing_type == 'MS':
# Load catalogues and templates
self.cat = self.Catalog(config=self.config, chip=chip, pointing=pointing, cat_dir=cat_dir, sed_dir=sed_dir, chip_output=chip_output)
self.cat = self.Catalog(config=self.config, chip=chip, pointing=pointing, cat_dir=cat_dir, sed_dir=sed_dir, chip_output=chip_output, filt=filt)
chip_output.create_output_file()
self.nobj = len(self.cat.objs)
......@@ -176,10 +151,33 @@ class Observation(object):
if temp_filter.filter_type.lower() == self.config["obs_setting"]["cut_in_band"].lower():
cut_filter = temp_filter
if self.config["ins_effects"]["field_dist"] == True:
self.fd_model = FieldDistortion(chip=chip)
else:
self.fd_model = None
# Loop over objects
missed_obj = 0
bright_obj = 0
dim_obj = 0
h_ext = generateExtensionHeader(
chip=chip,
xlen=chip.npix_x,
ylen=chip.npix_y,
ra=pointing.ra,
dec=pointing.dec,
pa=pointing.img_pa.deg,
gain=chip.gain,
readout=chip.read_noise,
dark=chip.dark_noise,
saturation=90000,
pixel_scale=chip.pix_scale,
pixel_size=chip.pix_size,
xcen=chip.x_cen,
ycen=chip.y_cen,
extName='raw')
for j in range(self.nobj):
# (DEBUG)
......@@ -187,61 +185,48 @@ class Observation(object):
# break
obj = self.cat.objs[j]
if obj.type == 'star' and self.config["run_option"]["galaxy_only"]:
continue
elif obj.type == 'galaxy' and self.config["run_option"]["star_only"]:
continue
elif obj.type == 'quasar' and self.config["run_option"]["star_only"]:
continue
# load and convert SED; also caculate object's magnitude in all CSST bands
try:
sed_data = self.cat.load_sed(obj)
norm_filt = self.cat.load_norm_filt(obj)
obj.sed, obj.param["mag_%s"%filt.filter_type], obj.param["flux_%s"%filt.filter_type] = self.cat.convert_sed(
obj.sed, obj.param["mag_%s"%filt.filter_type.lower()], obj.param["flux_%s"%filt.filter_type.lower()] = self.cat.convert_sed(
mag=obj.param["mag_use_normal"],
sed=sed_data,
target_filt=filt,
norm_filt=norm_filt,
)
_, obj.param["mag_%s"%cut_filter.filter_type], obj.param["flux_%s"%cut_filter.filter_type] = self.cat.convert_sed(
_, obj.param["mag_%s"%cut_filter.filter_type.lower()], obj.param["flux_%s"%cut_filter.filter_type.lower()] = self.cat.convert_sed(
mag=obj.param["mag_use_normal"],
sed=sed_data,
target_filt=cut_filter,
norm_filt=norm_filt,
)
except Exception as e:
# print(e)
traceback.print_exc()
chip_output.logger.error(e)
chip_output.Log_error(e)
continue
# chip_output.logger.info("debug point #1")
# [TODO] Testing
# chip_output.Log_info("mag_%s = %.3f"%(filt.filter_type.lower(), obj.param["mag_%s"%filt.filter_type.lower()]))
# Exclude very bright/dim objects (for now)
# if filt.is_too_bright(mag=obj.getMagFilter(filt)):
# if filt.is_too_bright(mag=obj.mag_use_normal):
if cut_filter.is_too_bright(
mag=obj.param["mag_%s"%self.config["obs_setting"]["cut_in_band"].lower()],
margin=self.config["obs_setting"]["mag_sat_margin"]):
# print("obj too birght!!", flush=True)
if obj.type != 'galaxy':
bright_obj += 1
obj.unload_SED()
continue
chip_output.Log_info("obj %s too birght!! mag_%s = %.3f"%(obj.id, cut_filter.filter_type, obj.param["mag_%s"%self.config["obs_setting"]["cut_in_band"].lower()]))
bright_obj += 1
obj.unload_SED()
continue
if filt.is_too_dim(
mag=obj.getMagFilter(filt),
margin=self.config["obs_setting"]["mag_lim_margin"]):
# if cut_filter.is_too_dim(mag=obj.param["mag_%s"%self.config["obs_setting"]["cut_in_band"].lower()]):
# print("obj too dim!!", flush=True)
chip_output.Log_info("obj %s too dim!! mag_%s = %.3f"%(obj.id, filt.filter_type, obj.getMagFilter(filt)))
dim_obj += 1
obj.unload_SED()
# print(obj.getMagFilter(filt))
continue
# chip_output.logger.info("debug point #2")
# Get corresponding shear values
if self.config["shear_setting"]["shear_type"] == "constant":
if obj.type == 'star':
obj.g1, obj.g2 = 0., 0.
......@@ -249,52 +234,34 @@ class Observation(object):
obj.g1, obj.g2 = self.g1_field, self.g2_field
elif self.config["shear_setting"]["shear_type"] == "extra":
try:
# TODO: every object with individual shear from input catalog(s)
# [TODO]: every object with individual shear from input catalog(s)
obj.g1, obj.g2 = self.g1_field[j], self.g2_field[j]
except:
# print("failed to load external shear.")
chip_output.logger.error("failed to load external shear.")
pass
# chip_output.logger.info("debug point #3")
chip_output.Log_error("failed to load external shear.")
elif self.config["shear_setting"]["shear_type"] == "catalog":
pass
else:
chip_output.logger.error("Unknown shear input")
chip_output.Log_error("Unknown shear input")
raise ValueError("Unknown shear input")
# chip_output.logger.info("debug point #4")
header_wcs = generateExtensionHeader(
xlen=chip.npix_x,
ylen=chip.npix_y,
ra=ra_cen,
dec=dec_cen,
pa=pointing.img_pa.deg,
gain=chip.gain,
readout=chip.read_noise,
dark=chip.dark_noise,
saturation=90000,
psize=chip.pix_scale,
row_num=chip.rowID,
col_num=chip.colID,
extName='raw')
pos_img, offset, local_wcs, real_wcs = obj.getPosImg_Offset_WCS(img=chip.img, fdmodel=self.fd_model, chip=chip, verbose=False, img_header=header_wcs)
# Get position of object on the focal plane
pos_img, offset, local_wcs, real_wcs, fd_shear = obj.getPosImg_Offset_WCS(img=chip.img, fdmodel=self.fd_model, chip=chip, verbose=False, img_header=h_ext)
# [TODO] For now, only consider objects which their centers (after field distortion) are projected within the focal plane
# Otherwise they will be considered missed objects
# if pos_img.x == -1 or pos_img.y == -1 or (not chip.isContainObj(x_image=pos_img.x, y_image=pos_img.y, margin=0.)):
if pos_img.x == -1 or pos_img.y == -1:
# Exclude object which is outside the chip area (after field distortion)
# print("obj missed!!")
chip_output.Log_info('obj_ra = %.6f, obj_dec = %.6f, obj_ra_orig = %.6f, obj_dec_orig = %.6f'%(obj.ra, obj.dec, obj.ra_orig, obj.dec_orig))
chip_output.Log_error("Objected missed: %s"%(obj.id))
missed_obj += 1
obj.unload_SED()
continue
# chip_output.logger.info("debug point #5")
# Draw object & update output catalog
try:
# chip_output.logger.info("debug point #6")
# chip_output.logger.info("current filter type: %s"%filt.filter_type)
if self.config["run_option"]["out_cat_only"]:
isUpdated = True
obj.real_pos = obj.getRealPos(chip.img, global_x=obj.posImg.x, global_y=obj.posImg.y, img_real_wcs=obj.real_wcs)
pos_shear = 0.
elif chip.survey_type == "photometric" and not self.config["run_option"]["out_cat_only"]:
isUpdated, pos_shear = obj.drawObj_multiband(
......@@ -306,8 +273,9 @@ class Observation(object):
chip=chip,
g1=obj.g1,
g2=obj.g2,
exptime=pointing.exp_time
)
exptime=pointing.exp_time,
fd_shear=fd_shear)
elif chip.survey_type == "spectroscopic" and not self.config["run_option"]["out_cat_only"]:
isUpdated, pos_shear = obj.drawObj_slitless(
tel=self.tel,
......@@ -319,30 +287,38 @@ class Observation(object):
g1=obj.g1,
g2=obj.g2,
exptime=pointing.exp_time,
normFilter=norm_filt)
# chip_output.logger.info("debug point #7")
if isUpdated:
normFilter=norm_filt,
fd_shear=fd_shear)
if isUpdated == 1:
# TODO: add up stats
# print("updating output catalog...")
chip_output.cat_add_obj(obj, pos_img, pos_shear)
pass
elif isUpdated == 0:
missed_obj += 1
chip_output.Log_error("Objected missed: %s"%(obj.id))
else:
# print("object omitted", flush=True)
chip_output.Log_error("Draw error, object omitted: %s"%(obj.id))
continue
except Exception as e:
# print(e)
traceback.print_exc()
chip_output.logger.error(e)
pass
chip_output.Log_error(e)
# # [C6 TEST]
# chip_output.Log_info("check running:1: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
# chip_output.Log_info('draw object %s'%obj.id)
# chip_output.Log_info('mag = %.3f'%obj.param['mag_use_normal'])
# Unload SED:
obj.unload_SED()
del obj
gc.collect()
del psf_model
del self.cat
gc.collect()
# print("check running:1: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True)
chip_output.logger.info("check running:1: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
chip_output.Log_info("check running:1: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
# Detector Effects
# ===========================================================
......@@ -357,6 +333,7 @@ class Observation(object):
ra_cen=pointing.ra,
dec_cen=pointing.dec,
img_rot=pointing.img_pa,
exptime=pointing.exp_time,
pointing_ID=pointing.id,
timestamp_obs=pointing.timestamp,
pointing_type=pointing.pointing_type,
......@@ -364,7 +341,7 @@ class Observation(object):
logger=chip_output.logger)
if pointing.pointing_type == 'MS':
datetime_obs = datetime.fromtimestamp(pointing.timestamp)
datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
date_obs = datetime_obs.strftime("%y%m%d")
time_obs = datetime_obs.strftime("%H%M%S")
h_prim = generatePrimaryHeader(
......@@ -373,45 +350,30 @@ class Observation(object):
pointNum = str(pointing.id),
ra=pointing.ra,
dec=pointing.dec,
psize=chip.pix_scale,
row_num=chip.rowID,
col_num=chip.colID,
pixel_scale=chip.pix_scale,
date=date_obs,
time_obs=time_obs,
exptime=pointing.exp_time,
im_type='SCI',
sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z],
sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz])
h_ext = generateExtensionHeader(
xlen=chip.npix_x,
ylen=chip.npix_y,
ra=pointing.ra,
dec=pointing.dec,
pa=pointing.img_pa.deg,
gain=chip.gain,
readout=chip.read_noise,
dark=chip.dark_noise,
saturation=90000,
psize=chip.pix_scale,
row_num=chip.rowID,
col_num=chip.colID,
extName='raw')
sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz],
chip_name=str(chip.chipID).rjust(2, '0'))
chip.img = galsim.Image(chip.img.array, dtype=np.uint16)
hdu1 = fits.PrimaryHDU(header=h_prim)
hdu1.add_checksum()
hdu2 = fits.ImageHDU(chip.img.array, header=h_ext)
hdu2.add_checksum()
hdu1 = fits.HDUList([hdu1, hdu2])
fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits')
hdu1.writeto(fname, output_verify='ignore', overwrite=True)
# print("# objects that are too bright %d out of %d"%(bright_obj, self.nobj))
# print("# objects that are too dim %d out of %d"%(dim_obj, self.nobj))
# print("# objects that are missed %d out of %d"%(missed_obj, self.nobj))
chip_output.logger.info("# objects that are too bright %d out of %d"%(bright_obj, self.nobj))
chip_output.logger.info("# objects that are too dim %d out of %d"%(dim_obj, self.nobj))
chip_output.logger.info("# objects that are missed %d out of %d"%(missed_obj, self.nobj))
chip_output.Log_info("# objects that are too bright %d out of %d"%(bright_obj, self.nobj))
chip_output.Log_info("# objects that are too dim %d out of %d"%(dim_obj, self.nobj))
chip_output.Log_info("# objects that are missed %d out of %d"%(missed_obj, self.nobj))
del chip.img
# print("check running:2: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True)
chip_output.logger.info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
chip_output.Log_info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
def runExposure_MPI_PointingList(self, pointing_list, shear_cat_file=None, chips=None, use_mpi=False):
if use_mpi:
......@@ -450,7 +412,7 @@ class Observation(object):
chip = run_chips[ichip]
filt = run_filts[ichip]
# print("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid), flush=True)
# chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid))
chip_output = ChipOutput(
config=self.config,
focal_plane=self.focal_plane,
......@@ -461,14 +423,13 @@ class Observation(object):
pointing_ID=pointing_ID,
subdir=sub_img_dir,
prefix=prefix)
chip_output.logger.info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid))
chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid))
self.run_one_chip(
chip=chip,
filt=filt,
chip_output=chip_output,
pointing=pointing,
cat_dir=self.path_dict["cat_dir"])
print("finished running chip#%d..."%(chip.chipID), flush=True)
chip_output.logger.info("finished running chip#%d..."%(chip.chipID))
pointing=pointing)
chip_output.Log_info("finished running chip#%d..."%(chip.chipID))
for handler in chip_output.logger.handlers[:]:
chip_output.logger.removeHandler(handler)
gc.collect()
......@@ -3,31 +3,46 @@ import numpy as np
class FieldDistortion(object):
def __init__(self, fdModel=None, fdModel_path=None):
def __init__(self, chip, fdModel=None, fdModel_path=None):
if fdModel is None:
import pickle
if fdModel_path is not None:
if hasattr(chip, 'fdModel'):
self.fdModel = chip.fdModel
elif fdModel_path is not None:
import pickle
with open(fdModel_path, "rb") as f:
self.fdModel = pickle.load(f)
else:
# with open("/data/simudata/CSSOSDataProductsSims/data/FieldDistModelv1.0.pickle", "rb") as f:
with open("/data/simudata/CSSOSDataProductsSims/data/FieldDistModelv2.0.pickle", "rb") as f:
self.fdModel = pickle.load(f)
raise ValueError("Error: no field distortion model has been specified!")
else:
self.fdModel = fdModel
self.ifdModel = self.fdModel["wave1"]
self.ixfdModel = self.ifdModel["xImagePos"]
self.iyfdModel = self.ifdModel["yImagePos"]
# first-order derivatives of the global field distortion model
self.ifx_dx = self.ixfdModel.partial_derivative(1,0)
self.ifx_dy = self.ixfdModel.partial_derivative(0,1)
self.ify_dx = self.iyfdModel.partial_derivative(1,0)
self.ify_dy = self.iyfdModel.partial_derivative(0,1)
if "residual" in self.fdModel["wave1"]:
self.irsModel = self.fdModel["wave1"]["residual"]["ccd" + chip.getChipLabel(chipID=chip.chipID)]
self.ixrsModel = self.irsModel["xResidual"]
self.iyrsModel = self.irsModel["yResidual"]
# first-order derivatives of the residual field distortion model
self.irx_dx = self.ixrsModel.partial_derivative(1,0)
self.irx_dy = self.ixrsModel.partial_derivative(0,1)
self.iry_dx = self.iyrsModel.partial_derivative(1,0)
self.iry_dy = self.iyrsModel.partial_derivative(0,1)
else:
self.irsModel = None
def isContainObj_FD(self, chip, pos_img):
# ifdModel = self.fdModel["ccd" + chip.getChipLabel(chipID=chip.chipID)]["wave1"]
# x, y = pos_img.x, pos_img.y
# xLowI, xUpI, yLowI, yUpI = ifdModel["InterpLim"]
ifdModel = self.fdModel["wave1"]
xLowI, xUpI, yLowI, yUpI = ifdModel["interpLimit"]
xLowI, xUpI, yLowI, yUpI = self.ifdModel["interpLimit"]
x, y = pos_img.x, pos_img.y
if (xLowI - x)*(xUpI - x) > 0 or (yLowI - y) * (yUpI - y) > 0:
if (xLowI - x) * (xUpI - x) > 0 or (yLowI - y) * (yUpI - y) > 0:
return False
return True
def get_Distorted(self, chip, pos_img, bandpass=None):
def get_distorted(self, chip, pos_img, bandpass=None):
""" Get the distored position for an undistorted image position
Parameters:
......@@ -42,19 +57,39 @@ class FieldDistortion(object):
the distored position.
"""
if not self.isContainObj_FD(chip=chip, pos_img=pos_img):
return galsim.PositionD(-1, -1)
# ifdModel = self.fdModel["ccd" + chip.getChipLabel(chipID=chip.chipID)]["wave1"]
ifdModel = self.fdModel["wave1"]
irsModel = self.fdModel["wave1"]["residual"]["ccd" + chip.getChipLabel(chipID=chip.chipID)]
return galsim.PositionD(-1, -1), None
x, y = pos_img.x, pos_img.y
x = ifdModel["xImagePos"](x, y)[0][0]
y = ifdModel["yImagePos"](x, y)[0][0]
x1LowI, x1UpI, y1LowI, y1UpI = irsModel["interpLimit"]
if (x1LowI-x)*(x1UpI-x) <=0 and (y1LowI-y)*(y1UpI-y)<=0:
dx = irsModel["xResidual"](x, y)[0][0]
dy = irsModel["yResidual"](x, y)[0][0]
x = self.ixfdModel(x, y)[0][0]
y = self.iyfdModel(x, y)[0][0]
ix_dx = self.ifx_dx(x, y)
ix_dy = self.ifx_dy(x, y)
iy_dx = self.ify_dx(x, y)
iy_dy = self.ify_dy(x, y)
if self.irsModel is not None:
# x1LowI, x1UpI, y1LowI, y1UpI = self.irsModel["interpLimit"]
# if (x1LowI-x)*(x1UpI-x) <=0 and (y1LowI-y)*(y1UpI-y)<=0:
# dx = self.ixrsModel(x, y)[0][0]
# dy = self.iyrsModel(x, y)[0][0]
# x += dx
# y += dy
# # field distortion induced ellipticity
# ix_dx = self.ifx_dx(x, y) + self.irx_dx(x, y)
# ix_dy = self.ifx_dy(x, y) + self.irx_dy(x, y)
# iy_dx = self.ify_dx(x, y) + self.iry_dx(x, y)
# iy_dy = self.ify_dy(x, y) + self.iry_dy(x, y)
# else:
# return galsim.PositionD(-1, -1), None
dx = self.ixrsModel(x, y)[0][0]
dy = self.iyrsModel(x, y)[0][0]
x += dx
y += dy
else:
return galsim.PositionD(-1, -1)
return galsim.PositionD(x, y)
# field distortion induced ellipticity
ix_dx = self.ifx_dx(x, y) + self.irx_dx(x, y)
ix_dy = self.ifx_dy(x, y) + self.irx_dy(x, y)
iy_dx = self.ify_dx(x, y) + self.iry_dx(x, y)
iy_dy = self.ify_dy(x, y) + self.iry_dy(x, y)
g1k_fd = 0.0 + (iy_dy - ix_dx) / (iy_dy + ix_dx)
g2k_fd = 0.0 - (iy_dx + ix_dy) / (iy_dy + ix_dx)
fd_shear = galsim.Shear(g1=g1k_fd, g2=g2k_fd)
return galsim.PositionD(x, y), fd_shear
......@@ -216,6 +216,7 @@ class PSFInterp(PSFModel):
self.sigGauss = psfRa
self.iccd = int(chip.getChipLabel(chipID=chip.chipID))
# self.iccd = chip.chip_name
if PSF_data_file == None:
print('Error - PSF_data_file is None')
sys.exit()
......@@ -256,7 +257,8 @@ class PSFInterp(PSFModel):
def _getPSFwave(self, iccd, PSF_data_file, PSF_data_prefix):
fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_ccd{:}.h5'.format(iccd), 'r')
# fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_ccd{:}.h5'.format(iccd), 'r')
fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_{:}.h5'.format(iccd), 'r')
nwave = len(fq.keys())
fq.close()
return nwave
......@@ -264,7 +266,8 @@ class PSFInterp(PSFModel):
def _loadPSF(self, iccd, PSF_data_file, PSF_data_prefix):
psfSet = []
fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_ccd{:}.h5'.format(iccd), 'r')
# fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_ccd{:}.h5'.format(iccd), 'r')
fq = h5py.File(PSF_data_file+'/' +PSF_data_prefix +'psfCube_{:}.h5'.format(iccd), 'r')
for ii in range(self.nwave):
iwave = ii+1
psfWave = []
......@@ -322,7 +325,7 @@ class PSFInterp(PSFModel):
"""
pixSize = np.rad2deg(self.pixsize*1e-3/28)*3600 #set psf pixsize
assert self.iccd == int(chip.getChipLabel(chipID=chip.chipID)), 'ERROR: self.iccd != chip.chipID'
# assert self.iccd == int(chip.getChipLabel(chipID=chip.chipID)), 'ERROR: self.iccd != chip.chipID'
twave = self._findWave(bandpass)
if twave == -1:
print("!!!PSF bandpass does not match.")
......
......@@ -12,10 +12,11 @@ def parse_args():
can be set in the .yaml config file as well.
'''
parser = argparse.ArgumentParser()
parser.add_argument('config_file', help='.yaml config file for simulation settings.')
parser.add_argument('-c', '--config_dir', help='Directory that houses the .yaml config file.')
parser.add_argument('-d', '--data_dir', help='Directory that houses the input data.')
parser.add_argument('-w', '--work_dir', help='The path for output.')
parser.add_argument('--config_file', type=str, required=True, help='.yaml config file for simulation settings.')
parser.add_argument('--catalog', type=str, required=True, help='name of the catalog interface class to be loaded.')
parser.add_argument('-c', '--config_dir', type=str, help='Directory that houses the .yaml config file.')
parser.add_argument('-d', '--data_dir', type=str, help='Directory that houses the input data.')
parser.add_argument('-w', '--work_dir', type=str, help='The path for output.')
return parser.parse_args()
def generate_pointing_list(config, pointing_filename=None, data_dir=None):
......@@ -144,7 +145,7 @@ def makeSubDir_PointingList(path_dict, config, pointing_ID=0):
return subImgdir, prefix
def get_shear_field(config, shear_cat_file=None):
if not config["shear_setting"]["shear_type"] in ["constant", "extra"]:
if not config["shear_setting"]["shear_type"] in ["constant", "extra", "catalog"]:
raise ValueError("Please set a right 'shear_method' parameter.")
if config["shear_setting"]["shear_type"] == "constant":
......@@ -152,16 +153,19 @@ def get_shear_field(config, shear_cat_file=None):
g2 = config["shear_setting"]["reduced_g2"]
nshear = 1
# TODO logging
else:
elif config["shear_setting"]["shear_type"] == "extra":
# TODO logging
if not os.path.exists(shear_cat_file):
raise ValueError("Cannot find shear catalog file.")
raise ValueError("Cannot find external shear catalog file.")
try:
shearCat = np.loadtxt(shear_cat_file)
nshear = shearCat.shape[0]
g1, g2 = shearCat[:, 0], shearCat[:, 1]
except:
print("Failed to the shear catalog file.")
print("Failed to read the shear catalog file.")
print("Setting to no shear.")
g1, g2 = 0., 0.
else:
g1, g2 = 0., 0.
nshear = 0
return g1, g2, nshear
\ No newline at end of file
......@@ -2,24 +2,32 @@
## 重要更新或问题修复:
* 2023.05.01: 更新至v2.0版本,新功能包含:
* C6仿真实现
* fits贴图功能
* 焦面探测器属性单独定义
* 无缝光谱探测器一度旋转
* 像场畸导致目标形变的仿真
* 导星探测器仿真
* 仪器参数更新,bug修复等
* 2022.08.23: 更新至最新的透过率曲线,在图像头文件中加入版本号。仿真软件版本号进行重新标记,当前版本为1.0.4
* 2022.06.20: 版本v0.5上线,请于release_v0.5 branch中进行clone操作(注:目前已重新设置为v1.0,release_v1.0.0)
* 2022.06.20: 版本v0.5上线,请于release_v0.5 branch中进行clone操作(注:目前已重新设置为v1.0,release_v1.0.0)
* 2022.04.13: 修复了天测模块中的调用错误
* 2022.04.08: 修复了仅输出星表选项("out_cat_only")存在的问题
## 使用方法及相关说明
* 软件安装和使用方法可参考:[软件说明文档](https://kdocs.cn/l/ckrtHibdV3OS)。目前还较为粗糙,持续更新中
* 仿真数据说明文档(见以上代码目录):CSST主巡天Cycle5模拟数据说明.pdf
<!---
* 仿真数据说明文档[C6数据说明文档](https://www.kdocs.cn/l/cuRmPf71Ly7v)
-->
* 在线版使用说明:[CSST网页接口使用说明](https://kdocs.cn/l/cl81flAXe1YQ)
* 软件安装及问题反馈表:[问题反馈表]https://f.kdocs.cn/w/XYcBkE63/
* 软件安装及问题反馈表:[问题反馈表](https://f.kdocs.cn/w/XYcBkE63/)
## 相关数据
* PSF采样及像场畸变数据下载:[百度云盘](https://pan.baidu.com/s/1oK2rwt9B1MVy1tyD3vx_gw),密码:i7yq
* PSF采样数据下载:[百度云盘](https://pan.baidu.com/s/1_2gbO6-d7x001_FMqZ1tfw),密码:2awt
* 滤光片及无缝光谱相关数据:[百度云盘](https://pan.baidu.com/s/1JROggg8IKf27i1DnwTaPAw),密码:07v5
......
......@@ -2,24 +2,16 @@
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2021/10/07
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
# work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/"
work_dir: "/share/home/fangyuedong/test_psf_rot/csst-simulation/workplace/"
work_dir: "/share/home/fangyuedong/fgs_sim/csst-simulation/workplace/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "PSF_TEST"
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_test_NGP_2.17.dat"
run_name: "C6_spectroscopic_20230225"
# Whether to use MPI
run_option:
......@@ -33,12 +25,35 @@ run_option:
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_C6_20221212"
star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
galaxy_cat: "cat2CSSTSim_bundle/"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Catalog_C6_20221212/sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
# Only simulate stars?
star_only: YES
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
......@@ -47,14 +62,14 @@ obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "All"
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
# (Subject to change)
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
......@@ -62,10 +77,16 @@ obs_setting:
# Note: NOT valid when a pointing list file is specified
ra_center: 192.8595
dec_center: 27.1283
# Image rotation [degree]
image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_radec_246.5_40.dat"
# Number of calibration pointings
np_cal: 0
......@@ -79,14 +100,13 @@ obs_setting:
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: null
run_chips: [24]
# Whether to enable astrometric modeling
# astrometric_lib: "libshao.so"
enable_astrometric_model: True
# Cut by saturation/limiting magnitude in which band?
cut_in_band: "g"
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
......@@ -94,20 +114,6 @@ obs_setting:
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# Input path setting
###############################################
# Default path settings for WIDE survey simulation
input_path:
cat_dir: "OnOrbitCalibration/CTargets20211231"
star_cat: "CT-NGP_r1.8_G28.hdf5"
galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Templates/Galaxy/"
###############################################
# PSF setting
###############################################
......@@ -125,13 +131,7 @@ psf_setting:
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube"
# path to field-distortion model
# Note: only valid when ins_effects: field_dist is "ON"
fd_path: "FieldDistModelGlobal_v1.0.pickle"
# sigma_spin: 0.0 # psf spin?
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
###############################################
# Shear setting
......@@ -140,19 +140,13 @@ psf_setting:
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog (not available yet)
# "catalog": from catalog
# "extra": from seprate file
shear_type: "constant"
shear_type: "catalog"
# For constant shear filed
reduced_g1: 0.026
reduced_g2: 0.015
# Representation of the shear vector?
reShear: "E"
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
reduced_g1: 0.
reduced_g2: 0.
# Extra shear catalog
# (currently not used)
......@@ -163,6 +157,8 @@ shear_setting:
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise
......@@ -183,20 +179,20 @@ ins_effects:
add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# values
dark_exptime: 300 # Exposure time for dark current frames [seconds]
flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
readout_time: 40 # The read-out time for each channel [seconds]
df_strength: 2.3 # Sillicon sensor diffusion strength
bias_level: 500 # bias level [e-/pixel]
gain: 1.1 # Gain
full_well: 90000 # Full well depth [e-]
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 40 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 500 # bias level [e-/pixel]
# gain: 1.1 # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
......@@ -205,12 +201,14 @@ output_setting:
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_Av: 121212 # Seed for generating random intrinsic extinction
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
......
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/share/simudata/CSSOSDataProductsSims/data/CSSTSimImage_C6/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "C6_fits_testRun"
# Whether to use MPI
run_option:
use_mpi: YES
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_C6_20221212"
star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
galaxy_cat: "cat2CSSTSim_bundle/"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
stamp_cat: "stampCatsIndex.hdf5"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Catalog_C6_20221212/sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
#stamp_SED:
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
# With stamp?
stamp_yes: YES
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 244.972773
dec_center: 39.895901
# Image rotation [degree]
image_rot: 109.59
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_radec_246.5_40.dat"
# pointing_dir: null
# pointing_file: null
# Number of calibration pointings
np_cal: 0
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [7]
# Whether to enable astrometric modeling
enable_astrometric_model: False
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog
# "extra": from seprate file
shear_type: "catalog"
# For constant shear filed
reduced_g1: 0.
reduced_g2: 0.
# Extra shear catalog
# (currently not used)
# shear_cat: "mockShear.cat"
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: OFF # Whether to add field distortions
add_back: OFF # Whether to add sky background
add_dark: OFF # Whether to add dark noise
add_readout: OFF # Whether to add read-out (Gaussian) noise
add_bias: OFF # Whether to add bias-level to images
bias_16channel: OFF # Whether to add different biases for 16 channels
gain_16channel: OFF # Whether to make different gains for 16 channels
shutter_effect: OFF # Whether to add shutter effect
flat_fielding: OFF # Whether to add flat-fielding effect
prnu_effect: OFF # Whether to add PRNU effect
non_linear: OFF # Whether to add non-linearity
cosmic_ray: OFF # Whether to add cosmic-ray
cray_differ: OFF # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: OFF # Whether to simulate CTE trails
saturbloom: OFF # Whether to simulate Saturation & Blooming
add_badcolumns: OFF # Whether to add bad columns
add_hotpixels: OFF # Whether to add hot pixels
add_deadpixels: OFF # Whether to add dead(dark) pixels
bright_fatter: OFF # Whether to simulate Brighter-Fatter (also diffusion) effect
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 0.01 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 2000 # bias level [e-/pixel]
# gain: 1. # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
......@@ -2,24 +2,16 @@
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2021/10/07
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
# work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/"
work_dir: "/public/home/fangyuedong/test/CSST/workplace/"
data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
run_name: "TEST_Speed"
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/data/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_test_NGP_2.17.dat"
work_dir: "/share/home/fangyuedong/test_psf_rot/csst-simulation/workplace/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "NGP_C3"
# Whether to use MPI
run_option:
......@@ -33,12 +25,32 @@ run_option:
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "OnOrbitCalibration/CTargets20211231"
star_cat: "CT-NGP_r1.8_G28.hdf5"
galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Templates/Galaxy/"
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
......@@ -47,14 +59,14 @@ obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "All"
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
# (Subject to change)
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
......@@ -62,10 +74,16 @@ obs_setting:
# Note: NOT valid when a pointing list file is specified
ra_center: 192.8595
dec_center: 27.1283
# Image rotation [degree]
image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_test_NGP_2.17.dat"
# Number of calibration pointings
np_cal: 0
......@@ -73,34 +91,25 @@ obs_setting:
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
run_pointings: [0, 1, 2]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [24]
run_chips: null
# Whether to enable astrometric modeling
# astrometric_lib: "libshao.so"
enable_astrometric_model: True
# Cut by saturation/limiting magnitude in which band?
cut_in_band: "g"
# Cut by saturation magnitude in which band?
cut_in_band: "z"
###############################################
# Input path setting
###############################################
# Default path settings for WIDE survey simulation
input_path:
cat_dir: "OnOrbitCalibration/CTargets20211231"
star_cat: "CT-NGP_r1.8_G28.hdf5"
galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5"
# saturation magnitude margin
mag_sat_margin: -2.5
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Templates/Galaxy/"
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
......@@ -119,13 +128,7 @@ psf_setting:
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube"
# path to field-distortion model
# Note: only valid when ins_effects: field_dist is "ON"
fd_path: "FieldDistModelGlobal_v1.0.pickle"
# sigma_spin: 0.0 # psf spin?
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube"
###############################################
# Shear setting
......@@ -134,7 +137,7 @@ psf_setting:
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog (not available yet)
# "catalog": from catalog
# "extra": from seprate file
shear_type: "constant"
......@@ -142,12 +145,6 @@ shear_setting:
reduced_g1: 0.026
reduced_g2: 0.015
# Representation of the shear vector?
reShear: "E"
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
# Extra shear catalog
# (currently not used)
# shear_cat: "mockShear.cat"
......@@ -157,38 +154,42 @@ shear_setting:
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise
add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images
shutter_effect: ON # Whether to add shutter effect
flat_fielding: ON # Whether to add flat-fielding effect
prnu_effect: ON # Whether to add PRNU effect
non_linear: ON # Whether to add non-linearity
cosmic_ray: ON # Whether to add cosmic-ray
cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: ON # Whether to simulate CTE trails
saturbloom: ON # Whether to simulate Saturation & Blooming
add_badcolumns: ON # Whether to add bad columns
add_hotpixels: ON # Whether to add hot pixels
add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# values
dark_exptime: 300 # Exposure time for dark current frames [seconds]
flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
readout_time: 40 # The read-out time for each channel [seconds]
df_strength: 2.3 # Sillicon sensor diffusion strength
bias_level: 500 # bias level [e-/pixel]
gain: 1.1 # Gain
full_well: 90000 # Full well depth [e-]
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Output options
add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images
bias_16channel: ON # Whether to add different biases for 16 channels
gain_16channel: ON # Whether to make different gains for 16 channels
shutter_effect: ON # Whether to add shutter effect
flat_fielding: ON # Whether to add flat-fielding effect
prnu_effect: ON # Whether to add PRNU effect
non_linear: ON # Whether to add non-linearity
cosmic_ray: ON # Whether to add cosmic-ray
cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: ON # Whether to simulate CTE trails
saturbloom: ON # Whether to simulate Saturation & Blooming
add_badcolumns: ON # Whether to add bad columns
add_hotpixels: ON # Whether to add hot pixels
add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 40 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 500 # bias level [e-/pixel]
# gain: 1.1 # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
......@@ -197,12 +198,14 @@ output_setting:
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_Av: 121212 # Seed for generating random intrinsic extinction
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
......
......@@ -2,24 +2,17 @@
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2021/10/07
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
# work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/"
work_dir: "/public/home/fangyuedong/20211203/CSST/workplace/"
data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
run_name: "example_20211217"
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: null
pointing_file: null
run_name: "example"
# Whether to use MPI
run_option:
......@@ -27,16 +20,29 @@ run_option:
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 40
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "catalog_points_7degree2/point_RA60.9624_DE-41.5032/"
star_cat: "stars_ccd13_p_RA60.9624_DE-41.5032.hdf5"
galaxy_cat: null
SED_templates_path:
star_SED: "SED_MMW_Gaia_Cluster_D20_SS.hdf5"
galaxy_SED: null
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
star_only: YES
###############################################
# Observation setting
......@@ -46,14 +52,14 @@ obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "All"
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
# (Subject to change)
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
......@@ -61,10 +67,16 @@ obs_setting:
# Note: NOT valid when a pointing list file is specified
ra_center: 60.9624
dec_center: -41.5032
# Image rotation [degree]
image_rot: -113.4333
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: null
pointing_file: null
# Number of calibration pointings
np_cal: 2
......@@ -80,24 +92,19 @@ obs_setting:
# Note: for all pointings
run_chips: [18]
# Cut by saturation/limiting magnitude in which band?
cut_in_band: "g"
# Whether to enable astrometric modeling
enable_astrometric_model: True
###############################################
# Input path setting
###############################################
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# Default path settings for WIDE survey simulation
input_path:
cat_dir: "catalog_points_7degree2/point_RA60.9624_DE-41.5032/"
star_cat: "stars_ccd13_p_RA60.9624_DE-41.5032.hdf5"
galaxy_cat: null
# saturation magnitude margin
mag_sat_margin: -2.5
SED_templates_path:
star_SED: "SED_MMW_Gaia_Cluster_D20_SS.hdf5"
galaxy_SED: null
# limiting magnitude margin
mag_lim_margin: +1.0
################################################
###############################################
# PSF setting
###############################################
psf_setting:
......@@ -116,12 +123,6 @@ psf_setting:
# NOTE: only valid for "Interp" PSF
psf_dir: "/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube"
# path to field-distortion model
# Note: only valid when ins_effects: field_dist is "ON"
fd_path: "FieldDistModelGlobal_v1.0.pickle"
# sigma_spin: 0.0 # psf spin?
###############################################
# Shear setting
###############################################
......@@ -129,7 +130,7 @@ psf_setting:
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog (not available yet)
# "catalog": from catalog
# "extra": from seprate file
shear_type: "constant"
......@@ -137,12 +138,6 @@ shear_setting:
reduced_g1: 0.026
reduced_g2: 0.015
# Representation of the shear vector?
reShear: "E"
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
# Extra shear catalog
# (currently not used)
# shear_cat: "mockShear.cat"
......@@ -152,11 +147,15 @@ shear_setting:
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise
add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images
bias_16channel: ON # Whether to add different biases for 16 channels
gain_16channel: ON # Whether to make different gains for 16 channels
shutter_effect: ON # Whether to add shutter effect
flat_fielding: ON # Whether to add flat-fielding effect
prnu_effect: ON # Whether to add PRNU effect
......@@ -170,20 +169,20 @@ ins_effects:
add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# values
dark_exptime: 300 # Exposure time for dark current frames [seconds]
flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
readout_time: 40 # The read-out time for each channel [seconds]
df_strength: 2.3 # Sillicon sensor diffusion strength
bias_level: 500 # bias level [e-/pixel]
gain: 1.1 # Gain
full_well: 90000 # Full well depth [e-]
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 40 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 500 # bias level [e-/pixel]
# gain: 1.1 # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
......@@ -192,12 +191,14 @@ output_setting:
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_Av: 121212 # Seed for generating random intrinsic extinction
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
......
......@@ -2,23 +2,16 @@
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2021/10/07
# CSST-Sim Group, 2023/04/25
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/public/home/fangyuedong/temp/CSST/workplace/"
data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
run_name: "TEST_random_knots"
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/data/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_test1214.dat"
work_dir: "/share/home/fangyuedong/fgs_sim/csst-simulation/workplace/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "fgs_test"
# Whether to use MPI
run_option:
......@@ -32,12 +25,35 @@ run_option:
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_C6_20221212"
star_cat: "C6_MMW_GGC_Astrometry_healpix.hdf5"
galaxy_cat: "cat2CSSTSim_bundle/"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Catalog_C6_20221212/sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
......@@ -46,24 +62,32 @@ obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "All"
survey_type: "FGS"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
# (Subject to change)
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 60.
dec_center: -40.
ra_center: 244.972773
dec_center: 39.895901
# Image rotation [degree]
image_rot: -113.4333
image_rot: 109.59
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
# pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
# pointing_file: "pointing_radec_246.5_40.dat"
pointing_dir: null
pointing_file: null
# Number of calibration pointings
np_cal: 0
......@@ -72,34 +96,25 @@ obs_setting:
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [1]
run_pointings: [0]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [1, 26, 29, 6, 7, 22, 23, 19, 20]
run_chips: [32]
# Whether to enable astrometric modeling
# astrometric_lib: "libshao.so"
enable_astrometric_model: True
# Cut by saturation/limiting magnitude in which band?
cut_in_band: "g"
###############################################
# Input path setting
###############################################
# Cut by saturation magnitude in which band?
cut_in_band: "FGS"
# Default path settings for WIDE survey simulation
input_path:
cat_dir: "Catalog_20210126/"
star_cat: "MMW_Cluster_D20_healpix_21.hdf5"
galaxy_cat: "galaxyCats_r_10.0_healpix.hdf5"
# saturation magnitude margin
mag_sat_margin: -2.5
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Templates/Galaxy/"
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
......@@ -118,13 +133,7 @@ psf_setting:
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube"
# path to field-distortion model
# Note: only valid when ins_effects: field_dist is "ON"
fd_path: "/data/simudata/CSSOSDataProductsSims/data/FieldDistModelGlobal_v1.0.pickle"
# sigma_spin: 0.0 # psf spin?
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
###############################################
# Shear setting
......@@ -133,19 +142,13 @@ psf_setting:
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog (not available yet)
# "catalog": from catalog
# "extra": from seprate file
shear_type: "constant"
shear_type: "catalog"
# For constant shear filed
reduced_g1: 0.026
reduced_g2: 0.015
# Representation of the shear vector?
reShear: "E"
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
reduced_g1: 0.
reduced_g2: 0.
# Extra shear catalog
# (currently not used)
......@@ -156,15 +159,19 @@ shear_setting:
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise
add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images
shutter_effect: ON # Whether to add shutter effect
bias_16channel: OFF # Whether to add different biases for 16 channels
gain_16channel: OFF # Whether to make different gains for 16 channels
shutter_effect: OFF # Whether to add shutter effect
flat_fielding: ON # Whether to add flat-fielding effect
prnu_effect: ON # Whether to add PRNU effect
non_linear: OFF # Whether to add non-linearity
non_linear: ON # Whether to add non-linearity
cosmic_ray: ON # Whether to add cosmic-ray
cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: ON # Whether to simulate CTE trails
......@@ -174,20 +181,20 @@ ins_effects:
add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# values
dark_exptime: 300 # Exposure time for dark current frames [seconds]
flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
readout_time: 40 # The read-out time for each channel [seconds]
df_strength: 2.3 # Sillicon sensor diffusion strength
bias_level: 500 # bias level [e-/pixel]
gain: 1.1 # Gain
full_well: 90000 # Full well depth [e-]
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 0.01 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 2000 # bias level [e-/pixel]
# gain: 1. # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
......@@ -196,12 +203,14 @@ output_setting:
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_Av: 121212 # Seed for generating random intrinsic extinction
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
......
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2023/04/30
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/share/simudata/CSSOSDataProductsSims/data/CSSTSimImage_C6/"
data_dir: "/share/home/weichengliang/CSST_git/csst-simulation/tools/"
run_name: "C6_fits_testRun"
# Whether to use MPI
run_option:
use_mpi: YES
# NOTE: "n_threads" paramters is currently not used in the backend
# simulation codes. It should be implemented later in the web frontend
# in order to config the number of threads to request from NAOC cluster
n_threads: 80
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
###############################################
# Catalog setting
###############################################
# Configure your catalog: options to be implemented
# in the corresponding (user defined) 'Catalog' class
catalog_options:
input_path:
cat_dir: "Catalog_test"
star_cat: "testStarCat.h5"
galaxy_cat: "testGalaxyCat.h5"
AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
stamp_cat: "stampCatsIndex.hdf5"
SED_templates_path:
star_SED: "SpecLib.hdf5"
galaxy_SED: "sedlibs/"
AGN_SED: "quickspeclib_ross13.fits"
AGN_SED_WAVE: "wave_ross13.npy"
#stamp_SED:
# simulate stars?
star_yes: NO
# simulate galaxies?
galaxy_yes: NO
# With stamp?
stamp_yes: YES
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 60.11828178499743 #244.972773
dec_center: -39.75897455697294 #39.895901
# Image rotation [degree]
image_rot: 112.23685624012192 #109.59
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: null #"/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: null #"pointing_radec_246.5_40.dat"
# Number of calibration pointings
np_cal: 0
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [7, 8, 9]
# Whether to enable astrometric modeling
enable_astrometric_model: False
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
# limiting magnitude margin
mag_lim_margin: +1.0
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog
# "extra": from seprate file
shear_type: "catalog"
# For constant shear filed
reduced_g1: 0.
reduced_g2: 0.
# Extra shear catalog
# (currently not used)
# shear_cat: "mockShear.cat"
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: ON # Whether to add field distortions
add_back: OFF # Whether to add sky background
add_dark: OFF # Whether to add dark noise
add_readout: OFF # Whether to add read-out (Gaussian) noise
add_bias: OFF # Whether to add bias-level to images
bias_16channel: OFF # Whether to add different biases for 16 channels
gain_16channel: OFF # Whether to make different gains for 16 channels
shutter_effect: OFF # Whether to add shutter effect
flat_fielding: OFF # Whether to add flat-fielding effect
prnu_effect: OFF # Whether to add PRNU effect
non_linear: OFF # Whether to add non-linearity
cosmic_ray: OFF # Whether to add cosmic-ray
cray_differ: OFF # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: OFF # Whether to simulate CTE trails
saturbloom: OFF # Whether to simulate Saturation & Blooming
add_badcolumns: OFF # Whether to add bad columns
add_hotpixels: OFF # Whether to add hot pixels
add_deadpixels: OFF # Whether to add dead(dark) pixels
bright_fatter: OFF # Whether to simulate Brighter-Fatter (also diffusion) effect
# Values:
# default values have been defined individually for each chip in:
# ObservationSim/Instrument/data/ccd/chip_definition.json
# Set them here will override the default values
# dark_exptime: 300 # Exposure time for dark current frames [seconds]
# flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
# readout_time: 0.01 # The read-out time for each channel [seconds]
# df_strength: 2.3 # Sillicon sensor diffusion strength
# bias_level: 2000 # bias level [e-/pixel]
# gain: 1. # Gain
# full_well: 90000 # Full well depth [e-]
###############################################
# Output options (for calibration pointings only)
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Random seeds
###############################################
random_seeds:
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
......@@ -2,4 +2,7 @@
date
python /public/home/fangyuedong/temp/CSST/run_example.py config_example.yaml -c /public/home/fangyuedong/temp/CSST/config
python3 /share/home/fangyuedong/fgs_sim/csst-simulation/run_sim.py \
--config_file config_example.yaml \
--catalog Catalog_example \
-c /share/home/fangyuedong/fgs_sim/csst-simulation/config
......@@ -16,4 +16,7 @@ NP=40
date
echo $NP
mpirun -np $NP python /public/home/fangyuedong/temp/CSST/run_sim.py config_example.yaml -c /public/home/fangyuedong/temp/CSST/config
mpirun -np $NP python /share/home/fangyuedong/fgs_sim/csst-simulation/run_sim.py \
--config_file config_example.yaml \
--catalog Catalog_example \
-c /share/home/fangyuedong/fgs_sim/csst-simulation/config
\ No newline at end of file
#!/bin/bash
#PBS -N SIMS
##PBS -l walltime=70:00:00
##mpdallexit
##mpdboot -n 10 -f ./mpd.hosts
##PBS -j oe
#PBS -l nodes=comput108:ppn=80
#####PBS -q longq
#PBS -q batch
#PBS -u fangyuedong
NP=80
date
echo $NP
mpirun -np $NP python /public/home/fangyuedong/temp/CSST/run_sim.py config_C3.yaml -c /public/home/fangyuedong/temp/CSST/config
#!/bin/bash
#PBS -N SIMS
#PBS -l nodes=wcl-1:ppn=60
###PBS -l nodes=wcl-1:ppn=24+wcl-2:ppn=24+wcl-3:ppn=24+wcl-4:ppn=24+wcl-5:ppn=24+wcl-6:ppn=24
#PBS -u fangyuedong
###PBS -j oe
cd $PBS_O_WORKDIR
NP=60
date
mpirun -np $NP python3 /share/home/fangyuedong/fgs_sim/csst-simulation/run_sim.py \
--config_file config_C6.yaml \
--catalog C6_Catalog \
-c /share/home/fangyuedong/fgs_sim/csst-simulation/config
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment