Commit 99b0497b authored by Fang Yuedong's avatar Fang Yuedong
Browse files

modified logic for defining exptime: use exptime defined in pointing as default

remove pointing_dir, data_dir in overall_config
parent 3da3cb66
......@@ -70,8 +70,7 @@ def get_star_cat(ra_pointing, dec_pointing):
class Catalog(CatalogBase):
def __init__(self, config, chip, pointing, chip_output, filt, **kwargs):
super().__init__()
self.cat_dir = os.path.join(config["data_dir"], config["catalog_options"]["input_path"]["cat_dir"])
# self.seed_Av = config["catalog_options"]["seed_Av"]
self.cat_dir = config["catalog_options"]["input_path"]["cat_dir"]
self.cosmo = FlatLambdaCDM(H0=67.66, Om0=0.3111)
......@@ -92,20 +91,19 @@ class Catalog(CatalogBase):
# Get the cloest star catalog file
star_file_name = get_star_cat(ra_pointing=self.pointing.ra, dec_pointing=self.pointing.dec)
star_path = os.path.join(config["catalog_options"]["input_path"]["star_cat"], star_file_name)
star_SED_file = config["catalog_options"]["SED_templates_path"]["star_SED"]
self.star_path = os.path.join(self.cat_dir, star_path)
self.star_SED_path = os.path.join(config["data_dir"], star_SED_file)
self.star_SED_path = config["catalog_options"]["SED_templates_path"]["star_SED"]
self._load_SED_lib_star()
if "galaxy_cat" in config["catalog_options"]["input_path"] and config["catalog_options"]["input_path"]["galaxy_cat"] and not config["catalog_options"]["star_only"]:
galaxy_dir = config["catalog_options"]["input_path"]["galaxy_cat"]
self.galaxy_path = os.path.join(self.cat_dir, galaxy_dir)
self.galaxy_SED_path = os.path.join(config["data_dir"], config["catalog_options"]["SED_templates_path"]["galaxy_SED"])
self.galaxy_SED_path = config["catalog_options"]["SED_templates_path"]["galaxy_SED"]
self._load_SED_lib_gals()
self.agn_seds = {}
if "AGN_SED" in config["catalog_options"]["SED_templates_path"] and not config["catalog_options"]["star_only"]:
self.AGN_SED_path = os.path.join(config["data_dir"], config["catalog_options"]["SED_templates_path"]["AGN_SED"])
self.AGN_SED_path = config["catalog_options"]["SED_templates_path"]["AGN_SED"]
if "rotateEll" in config["catalog_options"]:
self.rotation = np.radians(float(config["catalog_options"]["rotateEll"]))
......
......@@ -12,15 +12,19 @@ def config_dir(config, work_dir=None, data_dir=None):
path_dict["work_dir"] = work_dir
# Data directory
if data_dir == None:
# Assume all input datasets are in the work directory
path_dict["data_dir"] =os.path.join(path_dict["work_dir"], "data/")
else:
path_dict["data_dir"] = data_dir
# if data_dir == None:
# # Assume all input datasets are in the work directory
# path_dict["data_dir"] =os.path.join(path_dict["work_dir"], "data/")
# else:
# path_dict["data_dir"] = data_dir
# PSF data directory
# if config["psf_setting"]["psf_model"] == "Interp":
# path_dict["psf_pho_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_pho_dir"])
# path_dict["psf_sls_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_sls_dir"])
if config["psf_setting"]["psf_model"] == "Interp":
path_dict["psf_pho_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_pho_dir"])
path_dict["psf_sls_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_sls_dir"])
path_dict["psf_pho_dir"] = config["psf_setting"]["psf_pho_dir"]
path_dict["psf_sls_dir"] = config["psf_setting"]["psf_sls_dir"]
return path_dict
......
import numpy as np
import galsim
import os
import shutil
import yaml
import galsim
import numpy as np
from astropy.time import Time
from ObservationSim.Config._util import get_obs_id
import ObservationSim.Instrument._util as _util
class Pointing(object):
......@@ -103,3 +106,32 @@ class Pointing(object):
self.pointing_type = self.obs_param["obs_type"]
else:
self.timestamp = t
def make_output_pointing_dir(self, overall_config, pointing_ID=0, copy_obs_config=False):
run_dir = os.path.join(overall_config["work_dir"], overall_config["run_name"])
if not os.path.exists(run_dir):
try:
os.makedirs(run_dir, exist_ok=True)
except OSError:
pass
# self.prefix = "MSC_" + str(pointing_ID).rjust(8, '0')
self.output_prefix = get_obs_id(
img_type=self.pointing_type,
project_cycle=overall_config["project_cycle"],
run_counter=overall_config["run_counter"],
pointing_id=self.obs_id,
pointing_type_code = self.pointing_type_code)
self.output_dir = os.path.join(run_dir, self.output_prefix)
if not os.path.exists(self.output_dir):
try:
os.makedirs(self.output_dir, exist_ok=True)
except OSError:
pass
if copy_obs_config and self.obs_config_file:
obs_config_output_path = os.path.join(self.output_dir, os.path.basename(self.obs_config_file))
if not os.path.exists(obs_config_output_path):
try:
shutil.copy(args.obs_config_file, self.output_dir)
except OSError:
pass
......@@ -12,7 +12,7 @@ from ObservationSim.Config import config_dir, ChipOutput
from ObservationSim.Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip
from ObservationSim.Instrument.Chip import Effects
from ObservationSim.Instrument.Chip import ChipUtils as chip_utils
from ObservationSim._util import makeSubDir_PointingList
from ObservationSim._util import make_output_pointing_dir
from ObservationSim.Astrometry.Astrometry_util import on_orbit_obs_position
from ObservationSim.sim_steps import SimSteps, SIM_STEP_TYPES
......@@ -99,7 +99,6 @@ class Observation(object):
sim_steps = SimSteps(overall_config=self.config, chip_output=chip_output, all_filters=self.all_filters)
for step in pointing.obs_param["call_sequence"]:
print(step)
if self.config["run_option"]["out_cat_only"]:
if step != "scie_obs":
continue
......@@ -268,6 +267,9 @@ class Observation(object):
# Construct chips & filters:
pointing = pointing_list[ipoint]
pointing_ID = pointing.id
sub_img_dir, prefix = make_output_pointing_dir(path_dict=self.path_dict, config=self.config, pointing_ID=pointing_ID)
self.focal_plane = FocalPlane(chip_list=pointing.obs_param["run_chips"])
# Make Chip & Filter lists
self.chip_list = []
......@@ -310,8 +312,6 @@ class Observation(object):
continue
pid = os.getpid()
sub_img_dir, prefix = makeSubDir_PointingList(path_dict=self.path_dict, config=self.config, pointing_ID=pointing_ID)
chip = run_chips[ichip]
filt = run_filts[ichip]
# chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid))
......
......@@ -37,8 +37,12 @@ def generate_pointing_list(config, pointing_filename=None, data_dir=None):
else:
obs_config_file = None
if pointing_filename and data_dir:
# if pointing_filename and data_dir:
if pointing_filename:
if data_dir:
pointing_file = os.path.join(data_dir, pointing_filename)
else:
pointing_file = pointing_filename
f = open(pointing_file, 'r')
# for _ in range(1):
# header = f.readline()
......@@ -93,7 +97,7 @@ def make_run_dirs(work_dir, run_name, pointing_list):
pass
return imgDir
def makeSubDir_PointingList(path_dict, config, pointing_ID=0):
def make_output_pointing_dir(path_dict, config, pointing_ID=0):
imgDir = os.path.join(path_dict["work_dir"], config["run_name"])
if not os.path.exists(imgDir):
try:
......
......@@ -2,10 +2,17 @@ from ObservationSim.Instrument.Chip import ChipUtils as chip_utils
def add_cosmic_rays(self, chip, filt, tel, pointing, catalog, obs_param):
self.chip_output.Log_info(" Adding Cosmic-Ray")
# Get exposure time
if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None):
exptime = obs_param["exptime"]
else:
exptime = pointing.exp_time
chip.img, crmap_gsimg, cr_event_num = chip_utils.add_cosmic_rays(
img=chip.img,
chip=chip,
exptime=pointing.exp_time,
exptime=exptime,
seed=self.overall_config["random_seeds"]["seed_CR"]+pointing.id*30+chip.chipID)
# [TODO] output cosmic ray image
return chip, filt, tel, pointing
\ No newline at end of file
......@@ -12,6 +12,12 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
# Prepare output file(s) for this chip
self.chip_output.create_output_file()
# Get exposure time
if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None):
exptime = obs_param["exptime"]
else:
exptime = pointing.exp_time
# Load catalogues
if catalog is None:
self.chip_output.Log_error("Catalog interface class must be specified for SCIE-OBS")
......@@ -146,7 +152,7 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
chip=chip,
g1=obj.g1,
g2=obj.g2,
exptime=pointing.exp_time,
exptime=exptime,
fd_shear=fd_shear)
elif chip.survey_type == "spectroscopic" and not self.overall_config["run_option"]["out_cat_only"]:
......@@ -159,7 +165,7 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
chip=chip,
g1=obj.g1,
g2=obj.g2,
exptime=pointing.exp_time,
exptime=exptime,
normFilter=norm_filt,
fd_shear=fd_shear)
......
......@@ -11,20 +11,27 @@ def apply_PRNU(self, chip, filt, tel, pointing, catalog, obs_param):
def add_poisson_and_dark(self, chip, filt, tel, pointing, catalog, obs_param):
# Add dark current & Poisson noise
InputDark = False
# Get exposure time
if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None):
exptime = obs_param["exptime"]
else:
exptime = pointing.exp_time
if obs_param["add_dark"] == True:
if InputDark:
chip.img = chip_utils.add_inputdark(img=chip.img,
chip=chip,
exptime=pointing.exp_time)
exptime=exptime)
else:
chip.img, _ = chip_utils.add_poisson(img=chip.img,
chip=chip,
exptime=pointing.exp_time,
exptime=exptime,
poisson_noise=chip.poisson_noise)
else:
chip.img, _ = chip_utils.add_poisson(img=chip.img,
chip=self,
exptime=pointing.exp_time,
exptime=exptime,
poisson_noise=chip.poisson_noise,
dark_noise=0.)
return chip, filt, tel, pointing
......
......@@ -4,6 +4,13 @@ from ObservationSim.Straylight import calculateSkyMap_split_g
from ObservationSim.Instrument import FilterParam
def add_sky_background_sci(self, chip, filt, tel, pointing, catalog, obs_param):
# Get exposure time
if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None):
exptime = obs_param["exptime"]
else:
exptime = pointing.exp_time
flat_normal = np.ones_like(chip.img.array)
if obs_param["flat_fielding"] == True:
flat_normal = flat_normal * chip.flat_img.array / np.mean(chip.flat_img.array)
......@@ -25,7 +32,7 @@ def add_sky_background_sci(self, chip, filt, tel, pointing, catalog, obs_param):
self.chip_output.Log_info("sky background + stray light pixel flux value: %.5f"%(filt.sky_background))
if chip.survey_type == "photometric":
sky_map = filt.getSkyNoise(exptime = obs_param["exptime"])
sky_map = filt.getSkyNoise(exptime = exptime)
sky_map = sky_map * np.ones_like(chip.img.array) * flat_normal
sky_map = galsim.Image(array=sky_map)
else:
......@@ -39,7 +46,7 @@ def add_sky_background_sci(self, chip, filt, tel, pointing, catalog, obs_param):
isAlongY=0,
flat_cube=chip.flat_cube,
zoldial_spec = filt.zodical_spec)
sky_map = (sky_map + filt.sky_background)*obs_param["exptime"]
sky_map = (sky_map + filt.sky_background)*exptime
# sky_map = sky_map * tel.pupil_area * obs_param["exptime"]
chip.img += sky_map
......@@ -51,12 +58,17 @@ def add_sky_flat_calibration(self, chip, filt, tel, pointing, catalog, obs_param
_, _ = self.prepare_headers(chip=chip, pointing=pointing)
chip_wcs = galsim.FitsWCS(header = self.h_ext)
expTime = obs_param["exptime"]
# Get exposure time
if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None):
exptime = obs_param["exptime"]
else:
exptime = pointing.exp_time
skyback_level = obs_param["flat_level"]
filter_param = FilterParam()
sky_level_filt = obs_param["flat_level_filt"]
norm_scaler = skyback_level/expTime / filter_param.param[sky_level_filt][5]
norm_scaler = skyback_level/exptime / filter_param.param[sky_level_filt][5]
flat_normal = np.ones_like(chip.img.array)
if obs_param["flat_fielding"] == True:
......@@ -73,7 +85,7 @@ def add_sky_flat_calibration(self, chip, filt, tel, pointing, catalog, obs_param
if chip.survey_type == "photometric":
sky_map = flat_normal * np.ones_like(chip.img.array) * norm_scaler * filter_param.param[chip.filter_type][5] / tel.pupil_area * expTime
sky_map = flat_normal * np.ones_like(chip.img.array) * norm_scaler * filter_param.param[chip.filter_type][5] / tel.pupil_area * exptime
elif chip.survey_type == "spectroscopic":
# flat_normal = np.ones_like(chip.img.array)
if obs_param["flat_fielding"] == True:
......@@ -91,7 +103,7 @@ def add_sky_flat_calibration(self, chip, filt, tel, pointing, catalog, obs_param
pixelSize=chip.pix_scale,
isAlongY=0,
flat_cube=chip.flat_cube)
sky_map = sky_map * norm_scaler * expTime
sky_map = sky_map * norm_scaler * exptime
chip.img += sky_map
return chip, filt, tel, pointing
......
......@@ -11,7 +11,7 @@
# can add some of the command-line arguments here as well;
# ok to pass either way or both, as long as they are consistent
work_dir: "/public/home/fangyuedong/project/workplace/"
data_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/"
# data_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/"
run_name: "QSO_50sqdeg_test"
# Project cycle and run counter are used to name the outputs
......@@ -35,15 +35,15 @@ run_option:
catalog_options:
input_path:
# cat_dir: "Catalog_C6_20221212"
cat_dir: ""
cat_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/"
star_cat: "starcat/"
galaxy_cat: "qsocat/cat2CSSTSim_bundle-50sqDeg/"
# AGN_cat: "AGN_C6_ross13_rand_pos_rmax-1.3.fits"
SED_templates_path:
star_SED: "SpecLib.hdf5"
galaxy_SED: "sedlibs/"
AGN_SED: "qsocat/qsosed/"
star_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/SpecLib.hdf5"
galaxy_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/sedlibs/"
AGN_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/qsocat/qsosed/"
# AGN_SED_WAVE: "wave_ross13.npy"
# Only simulate stars?
......@@ -52,11 +52,6 @@ catalog_options:
# Only simulate galaxies?
galaxy_only: YES
# # rotate galaxy ellipticity
# rotateEll: 0. # [degree]
# seed_Av: 121212 # Seed for generating random intrinsic extinction
###############################################
# Observation setting
###############################################
......@@ -65,8 +60,8 @@ obs_setting:
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg"
pointing_file: "pointing_50_combined.dat"
# pointing_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/pointing50_C9/"
pointing_file: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/pointing50_C9/pointing_50_1_n.dat"
obs_config_file: "/public/home/fangyuedong/project/csst-simulation/config/obs_config_SCI_WIDE_phot.yaml"
......
......@@ -11,7 +11,7 @@
# Observation type
obs_type: "SCI"
obs_type_code: "101"
obs_id: "00000001"
obs_id: "00000001" # this setting will only be used if pointing list file is not given
# Define list of chips
run_chips: [8]
......@@ -20,27 +20,37 @@ run_chips: [8]
call_sequence:
# Accumulate fluxes from objects
scie_obs:
exptime: 150. # [s]
# [Optional]: exposure time of the pointing will be used as default.
# Set it here is you want to override the default
# exptime: 150. # [s]
shutter_effect: YES
flat_fielding: YES
field_dist: YES
# Accumulate fluxes from sky background
# flat_level: set the total skybackground value (e-) in the exptime,if none,set null, or delete the key
# flat_level_filt: the vale of "flat_level" is in the filter "flat_level_filt", can set NUV, u, g, r, i, z, y, if
# none,set null,or delete the key
sky_background:
exptime: 150. # [s]
# [Optional]: exposure time of the pointing will be used as default.
# Set it here is you want to override the default
# exptime: 150. # [s]
shutter_effect: YES
flat_fielding: YES
enable_straylight_model: YES
# flat_level: set the total skybackground value (e-) in the exptime,if none,set null, or delete the key
# flat_level_filt: the vale of "flat_level" is in the filter "flat_level_filt", can set NUV, u, g, r, i, z, y, if
# none,set null,or delete the key
flat_level: null
flat_level_filt: null
# Apply PRNU to accumulated photons
PRNU_effect: {}
# Accumulate photons caused by cosmic rays
cosmic_rays: {}
cosmic_rays:
# [Optional]: exposure time of the pointing will be used as default.
# Set it here is you want to override the default
# exptime: 150. # [s]
# Add Poission noise and dark current
poisson_and_dark:
# [Optional]: exposure time of the pointing will be used as default.
# Set it here is you want to override the default
# exptime: 150. # [s]
add_dark: YES
# Simulate brighter fatter effects
bright_fatter: {}
......
......@@ -49,6 +49,9 @@ def run_sim():
if args.work_dir is not None:
config['work_dir'] = args.work_dir
if not ("data_dir" in config):
config["data_dir"] = None
# Some default values
if "mag_sat_margin" not in config["obs_setting"]:
config["obs_setting"]["mag_sat_margin"] = -2.5
......@@ -64,7 +67,10 @@ def run_sim():
# "config['obs_setting']['np_cal']"" is the number of CAL pointings which will be
# appended to the front.
# NOTE: the implementation of gerenating time_stamps is temporary.
pointing_list = generate_pointing_list(config=config, pointing_filename=config['obs_setting']['pointing_file'], data_dir=config['obs_setting']['pointing_dir'])
pointing_dir = None
if "pointing_dir" in config['obs_setting']:
pointing_dir = config['obs_setting']["pointing_dir"]
pointing_list = generate_pointing_list(config=config, pointing_filename=config['obs_setting']['pointing_file'], data_dir=pointing_dir)
# Make the main output directories
run_dir = make_run_dirs(work_dir=config['work_dir'], run_name=config['run_name'], pointing_list=pointing_list)
......
......@@ -24,7 +24,7 @@ def defineCCD(iccd, config_file):
# print (key + " : " + str(value))
except yaml.YAMLError as exc:
print(exc)
path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir'])
# path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir'])
chip = Chip(chipID=iccd, config=config)
#chip = Chip(chipID=iccd, ccdEffCurve_dir=path_dict["ccd_dir"], CRdata_dir=path_dict["CRdata_dir"], normalize_dir=path_dict["normalize_dir"], sls_dir=path_dict['sls_dir'], config=config)
return chip
......
......@@ -21,7 +21,7 @@ def defineCCD(iccd, config_file):
# print (key + " : " + str(value))
except yaml.YAMLError as exc:
print(exc)
path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir'])
# path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir'])
chip = Chip(chipID=iccd, config=config)
#chip = Chip(chipID=iccd, ccdEffCurve_dir=path_dict["ccd_dir"], CRdata_dir=path_dict["CRdata_dir"], normalize_dir=path_dict["normalize_dir"], sls_dir=path_dict['sls_dir'], config=config)
return chip
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment