Skip to content
GitLab
Commit 3a96ec26 authored by Fang Yuedong's avatar Fang Yuedong
Browse files

add log for each treahds, add astrometry for pointings, add Catalog class for NGP fields

parent 931e5956
Hide whitespace changes
Inline Side-by-side
Catalog/C3Catalog.py
View file @ 3a96ec26
......@@ -28,6 +28,11 @@ class C3Catalog(CatalogBase):
self.cat_dir = os.path.join(config["data_dir"], config["input_path"]["cat_dir"])
self.seed_Av = config["random_seeds"]["seed_Av"]
if "logger" in kwargs:
self.logger = kwargs["logger"]
else:
self.logger = None
with pkg_resources.path('Catalog.data', 'SLOAN_SDSS.g.fits') as filter_path:
self.normF_star = Table.read(str(filter_path))
with pkg_resources.path('Catalog.data', 'lsst_throuput_g.fits') as filter_path:
......@@ -63,7 +68,11 @@ class C3Catalog(CatalogBase):
dec = np.deg2rad(np.array([dec_max, dec_max, dec_min, dec_min]))
vertices = spherical_to_cartesian(1., dec, ra)
self.pix_list = hp.query_polygon(NSIDE, np.array(vertices).T, inclusive=True)
print("HEALPix List: ", self.pix_list)
if self.logger is not None:
msg = str(("HEALPix List: ", self.pix_list))
self.logger.info(msg)
else:
print("HEALPix List: ", self.pix_list)
def load_norm_filt(self, obj):
if obj.type == "star":
......@@ -169,10 +178,10 @@ class C3Catalog(CatalogBase):
param['id'] = gals['galaxyID'][igals]
if param['star'] == 0:
obj = Galaxy(param, self.rotation)
obj = Galaxy(param, self.rotation, logger=self.logger)
self.objs.append(obj)
if param['star'] == 2:
obj = Quasar(param)
obj = Quasar(param, logger=self.logger)
self.objs.append(obj)
def _load_stars(self, stars, pix_id=None):
......@@ -230,7 +239,7 @@ class C3Catalog(CatalogBase):
param['feh'] = stars['feh'][istars]
param['z'] = 0.0
param['star'] = 1 # Star
obj = Star(param)
obj = Star(param, logger=self.logger)
self.objs.append(obj)
def _load(self, **kwargs):
......@@ -250,7 +259,10 @@ class C3Catalog(CatalogBase):
gals = gals_cat[str(pix)]
self._load_gals(gals, pix_id=pix)
del gals
print("number of objects in catalog: ", len(self.objs))
if self.logger is not None:
self.logger.info("number of objects in catalog: %d"%(len(self.objs)))
else:
print("number of objects in catalog: ", len(self.objs))
del self.avGal
......
Catalog/NGPCatalog.py 0 → 100644
View file @ 3a96ec26
import os
import galsim
import random
import numpy as np
import h5py as h5
import healpy as hp
import astropy.constants as cons
from astropy.coordinates import spherical_to_cartesian
from astropy.table import Table
from scipy import interpolate
from datetime import datetime
from ObservationSim.MockObject import CatalogBase, Star, Galaxy, Quasar
from ObservationSim.MockObject._util import seds, sed_assign, extAv, tag_sed, getObservedSED
from ObservationSim.Astrometry.Astrometry_util import on_orbit_obs_position
try:
import importlib.resources as pkg_resources
except ImportError:
# Try backported to PY<37 'importlib_resources'
import importlib_resources as pkg_resources
NSIDE = 128
class NGPCatalog(CatalogBase):
def __init__(self, config, chip, pointing, **kwargs):
super().__init__()
self.cat_dir = os.path.join(config["data_dir"], config["input_path"]["cat_dir"])
self.seed_Av = config["random_seeds"]["seed_Av"]
if "logger" in kwargs:
self.logger = kwargs["logger"]
else:
self.logger = None
with pkg_resources.path('Catalog.data', 'SLOAN_SDSS.g.fits') as filter_path:
self.normF_star = Table.read(str(filter_path))
with pkg_resources.path('Catalog.data', 'lsst_throuput_g.fits') as filter_path:
self.normF_galaxy = Table.read(str(filter_path))
self.config = config
self.chip = chip
self.pointing = pointing
if "star_cat" in config["input_path"] and config["input_path"]["star_cat"] and not config["run_option"]["galaxy_only"]:
star_file = config["input_path"]["star_cat"]
star_SED_file = config["SED_templates_path"]["star_SED"]
self.star_path = os.path.join(self.cat_dir, star_file)
self.star_SED_path = os.path.join(config["data_dir"], star_SED_file)
self._load_SED_lib_star()
if "galaxy_cat" in config["input_path"] and config["input_path"]["galaxy_cat"] and not config["run_option"]["star_only"]:
galaxy_file = config["input_path"]["galaxy_cat"]
self.galaxy_path = os.path.join(self.cat_dir, galaxy_file)
self.galaxy_SED_path = os.path.join(config["data_dir"], config["SED_templates_path"]["galaxy_SED"])
self._load_SED_lib_gals()
if "rotateEll" in config["shear_setting"]:
self.rotation = float(int(config["shear_setting"]["rotateEll"]/45.))
else:
self.rotation = 0.
self._get_healpix_list()
self._load()
def _get_healpix_list(self):
self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2)
ra_min, ra_max, dec_min, dec_max = self.sky_coverage.xmin, self.sky_coverage.xmax, self.sky_coverage.ymin, self.sky_coverage.ymax
ra = np.deg2rad(np.array([ra_min, ra_max, ra_max, ra_min]))
dec = np.deg2rad(np.array([dec_max, dec_max, dec_min, dec_min]))
vertices = spherical_to_cartesian(1., dec, ra)
self.pix_list = hp.query_polygon(NSIDE, np.array(vertices).T, inclusive=True)
if self.logger is not None:
msg = str(("HEALPix List: ", self.pix_list))
self.logger.info(msg)
else:
print("HEALPix List: ", self.pix_list)
def load_norm_filt(self, obj):
if obj.type == "star":
return self.normF_star
elif obj.type == "galaxy" or obj.type == "quasar":
return self.normF_galaxy
else:
return None
def _load_SED_lib_star(self):
self.tempSED_star = h5.File(self.star_SED_path,'r')
def _load_SED_lib_gals(self):
self.tempSed_gal, self.tempRed_gal = seds("galaxy.list", seddir=self.galaxy_SED_path)
def _load_gals(self, gals, pix_id=None):
ngals = len(gals['galaxyID'])
self.rng_sedGal = random.Random()
self.rng_sedGal.seed(pix_id) # Use healpix index as the random seed
self.ud = galsim.UniformDeviate(pix_id)
# Apply astrometric modeling
# in C3 case only aberration
ra_arr = gals['ra_true'][:]
dec_arr = gals['dec_true'][:]
if self.config["obs_setting"]["enable_astrometric_model"]:
ra_list = ra_arr.tolist()
dec_list = dec_arr.tolist()
pmra_list = np.zeros(ngals).tolist()
pmdec_list = np.zeros(ngals).tolist()
rv_list = np.zeros(ngals).tolist()
parallax_list = [1e-9] * ngals
dt = datetime.fromtimestamp(self.pointing.timestamp)
date_str = dt.date().isoformat()
time_str = dt.time().isoformat()
ra_arr, dec_arr = on_orbit_obs_position(
input_ra_list=ra_list,
input_dec_list=dec_list,
input_pmra_list=pmra_list,
input_pmdec_list=pmdec_list,
input_rv_list=rv_list,
input_parallax_list=parallax_list,
input_nstars=ngals,
input_x=self.pointing.sat_x,
input_y=self.pointing.sat_y,
input_z=self.pointing.sat_z,
input_vx=self.pointing.sat_vx,
input_vy=self.pointing.sat_vy,
input_vz=self.pointing.sat_vz,
input_epoch="J2015.5",
input_date_str=date_str,
input_time_str=time_str
)
for igals in range(ngals):
param = self.initialize_param()
param['ra'] = ra_arr[igals]
param['dec'] = dec_arr[igals]
param['ra_orig'] = gals['ra_true'][igals]
param['dec_orig'] = gals['dec_true'][igals]
if not self.chip.isContainObj(ra_obj=param['ra'], dec_obj=param['dec'], margin=200):
continue
param['mag_use_normal'] = gals['mag_true_g_lsst'][igals]
if param['mag_use_normal'] >= 26.5:
continue
param['z'] = gals['redshift_true'][igals]
param['model_tag'] = 'None'
param['gamma1'] = 0
param['gamma2'] = 0
param['kappa'] = 0
param['delta_ra'] = 0
param['delta_dec'] = 0
# sersicB = gals['sersic_bulge'][igals]
hlrMajB = gals['size_bulge_true'][igals]
hlrMinB = gals['size_minor_bulge_true'][igals]
# sersicD = gals['sersic_disk'][igals]
hlrMajD = gals['size_disk_true'][igals]
hlrMinD = gals['size_minor_disk_true'][igals]
aGal = gals['size_true'][igals]
bGal = gals['size_minor_true'][igals]
param['bfrac'] = gals['bulge_to_total_ratio_i'][igals]
param['theta'] = gals['position_angle_true'][igals]
param['hlr_bulge'] = np.sqrt(hlrMajB * hlrMinB)
param['hlr_disk'] = np.sqrt(hlrMajD * hlrMinD)
param['ell_bulge'] = (hlrMajB - hlrMinB)/(hlrMajB + hlrMinB)
param['ell_disk'] = (hlrMajD - hlrMinD)/(hlrMajD + hlrMinD)
param['ell_tot'] = (aGal - bGal) / (aGal + bGal)
# Assign each galaxy a template SED
param['sed_type'] = sed_assign(phz=param['z'], btt=param['bfrac'], rng=self.rng_sedGal)
param['redden'] = self.tempRed_gal[param['sed_type']]
param['av'] = self.avGal[int(self.ud()*self.nav)]
if param['sed_type'] <= 5:
param['av'] = 0.0
param['redden'] = 0
param['star'] = 0 # Galaxy
if param['sed_type'] >= 29:
param['av'] = 0.6 * param['av'] / 3.0 # for quasar, av=[0, 0.2], 3.0=av.max-av.im
param['star'] = 2 # Quasar
self.ids += 1
# param['id'] = self.ids
param['id'] = gals['galaxyID'][igals]
if param['star'] == 0:
obj = Galaxy(param, self.rotation, logger=self.logger)
self.objs.append(obj)
if param['star'] == 2:
obj = Quasar(param, logger=self.logger)
self.objs.append(obj)
def _load_stars(self, stars, pix_id=None):
nstars = len(stars['sourceID'])
# Apply astrometric modeling
ra_arr = stars["RA"][:]
dec_arr = stars["Dec"][:]
pmra_arr = stars['pmra'][:]
pmdec_arr = stars['pmdec'][:]
rv_arr = stars['RV'][:]
parallax_arr = stars['parallax'][:]
if self.config["obs_setting"]["enable_astrometric_model"]:
ra_list = ra_arr.tolist()
dec_list = dec_arr.tolist()
pmra_list = pmra_arr.tolist()
pmdec_list = pmdec_arr.tolist()
rv_list = rv_arr.tolist()
parallax_list = parallax_arr.tolist()
dt = datetime.fromtimestamp(self.pointing.timestamp)
date_str = dt.date().isoformat()
time_str = dt.time().isoformat()
ra_arr, dec_arr = on_orbit_obs_position(
input_ra_list=ra_list,
input_dec_list=dec_list,
input_pmra_list=pmra_list,
input_pmdec_list=pmdec_list,
input_rv_list=rv_list,
input_parallax_list=parallax_list,
input_nstars=nstars,
input_x=self.pointing.sat_x,
input_y=self.pointing.sat_y,
input_z=self.pointing.sat_z,
input_vx=self.pointing.sat_vx,
input_vy=self.pointing.sat_vy,
input_vz=self.pointing.sat_vz,
input_epoch="J2015.5",
input_date_str=date_str,
input_time_str=time_str
)
for istars in range(nstars):
param = self.initialize_param()
param['ra'] = ra_arr[istars]
param['dec'] = dec_arr[istars]
param['ra_orig'] = stars["RA"][istars]
param['dec_orig'] = stars["Dec"][istars]
param['pmra'] = pmra_arr[istars]
param['pmdec'] = pmdec_arr[istars]
param['rv'] = rv_arr[istars]
param['parallax'] = parallax_arr[istars]
if not self.chip.isContainObj(ra_obj=param['ra'], dec_obj=param['dec'], margin=200):
continue
param['mag_use_normal'] = stars['app_sdss_g'][istars]
if param['mag_use_normal'] >= 26.5:
continue
self.ids += 1
# param['id'] = self.ids
param['id'] = stars['sourceID'][istars]
param['sed_type'] = stars['sourceID'][istars]
param['model_tag'] = stars['model_tag'][istars]
param['teff'] = stars['teff'][istars]
param['logg'] = stars['grav'][istars]
param['feh'] = stars['feh'][istars]
param['z'] = 0.0
param['star'] = 1 # Star
obj = Star(param, logger=self.logger)
self.objs.append(obj)
def _load(self, **kwargs):
self.nav = 15005
self.avGal = extAv(self.nav, seed=self.seed_Av)
self.objs = []
self.ids = 0
if "star_cat" in self.config["input_path"] and self.config["input_path"]["star_cat"] and not self.config["run_option"]["galaxy_only"]:
star_cat = h5.File(self.star_path, 'r')['catalog']
for pix in self.pix_list:
stars = star_cat[str(pix)]
self._load_stars(stars, pix_id=pix)
del stars
if "galaxy_cat" in self.config["input_path"] and self.config["input_path"]["galaxy_cat"] and not self.config["run_option"]["star_only"]:
gals_cat = h5.File(self.galaxy_path, 'r')['galaxies']
for pix in self.pix_list:
gals = gals_cat[str(pix)]
self._load_gals(gals, pix_id=pix)
del gals
if self.logger is not None:
self.logger.info("number of objects in catalog: %d"%(len(self.objs)))
else:
print("number of objects in catalog: ", len(self.objs))
del self.avGal
def load_sed(self, obj, **kwargs):
if obj.type == 'star':
_, wave, flux = tag_sed(
h5file=self.tempSED_star,
model_tag=obj.param['model_tag'],
teff=obj.param['teff'],
logg=obj.param['logg'],
feh=obj.param['feh']
)
elif obj.type == 'galaxy' or obj.type == 'quasar':
sed_data = getObservedSED(
sedCat=self.tempSed_gal[obj.sed_type],
redshift=obj.z,
av=obj.param["av"],
redden=obj.param["redden"]
)
wave, flux = sed_data[0], sed_data[1]
else:
raise ValueError("Object type not known")
speci = interpolate.interp1d(wave, flux)
# lamb = np.arange(2500, 10001 + 0.5, 0.5)
lamb = np.arange(2400, 11001 + 0.5, 0.5)
y = speci(lamb)
# erg/s/cm2/A --> photo/s/m2/A
all_sed = y * lamb / (cons.h.value * cons.c.value) * 1e-13
sed = Table(np.array([lamb, all_sed]).T, names=('WAVELENGTH', 'FLUX'))
del wave
del flux
return sed
ObservationSim/Config/ChipOutput.py
View file @ 3a96ec26
import os
import logging
class ChipOutput(object):
def __init__(self, config, focal_plane, chip, filt, imgKey0="", imgKey1="", imgKey2="", exptime=150., mjdTime="", ra_cen=None, dec_cen=None, pointing_type='MS', pointing_ID='0', subdir="./", prefix=""):
......@@ -20,9 +21,22 @@ class ChipOutput(object):
self.chipLabel = focal_plane.getChipLabel(chip.chipID)
self.img_name = prefix + exp_name%(self.chipLabel, filt.filter_type)
self.cat_name = 'MSC_' + config["obs_setting"]["date_obs"] + config["obs_setting"]["time_obs"] + "_" + str(pointing_ID).rjust(7, '0') + "_" + self.chipLabel.rjust(2,'0') + ".cat"
# self.cat_name = 'MSC_' + config["obs_setting"]["date_obs"] + config["obs_setting"]["time_obs"] + "_" + str(pointing_ID).rjust(7, '0') + "_" + self.chipLabel.rjust(2,'0') + ".cat"
self.cat_name = "MSC_%s_chip_%s_filt_%s"%(str(pointing_ID).rjust(7, '0'), focal_plane.getChipLabel(chip.chipID), filt.filter_type) + ".cat"
self.subdir = subdir
# Setup logger for each chip
logger_filename = "MSC_%s_chip_%s_filt_%s"%(str(pointing_ID).rjust(7, '0'), focal_plane.getChipLabel(chip.chipID), filt.filter_type) + ".log"
self.logger = logging.getLogger()
fh = logging.FileHandler(os.path.join(self.subdir, logger_filename), mode='w+', encoding='utf-8')
fh.setLevel(logging.DEBUG)
self.logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
self.logger.addHandler(fh)
hdr1 = "obj_ID ID_chip filter xImage yImage ra dec ra_orig dec_orig z mag obj_type "
hdr2 = "thetaR bfrac hlr_disk hlr_bulge e1_disk e2_disk e1_bulge e2_bulge g1 g2 "
hdr3 = "sed_type av redden "
......@@ -36,10 +50,10 @@ class ChipOutput(object):
self.hdr = hdr1 + hdr2 + hdr3 + hdr4
self.fmt = fmt1 + fmt2 + fmt3 + fmt4
print("pointing_type = %s\n"%(pointing_type))
self.logger.info("pointing_type = %s\n"%(pointing_type))
if pointing_type == 'MS':
self.cat = open(os.path.join(self.subdir, self.cat_name), "w")
print("Creating catalog file %s ...\n"%(os.path.join(self.subdir, self.cat_name)))
self.logger.info("Creating catalog file %s ...\n"%(os.path.join(self.subdir, self.cat_name)))
self.cat.write(self.hdr)
# def updateHDR(self, hdr):
......
ObservationSim/Instrument/Chip/Chip.py
View file @ 3a96ec26
......@@ -17,7 +17,7 @@ except ImportError:
import importlib_resources as pkg_resources
class Chip(FocalPlane):
def __init__(self, chipID, ccdEffCurve_dir=None, CRdata_dir=None, sls_dir=None, config=None, treering_func=None):
def __init__(self, chipID, ccdEffCurve_dir=None, CRdata_dir=None, sls_dir=None, config=None, treering_func=None, logger=None):
# Get focal plane (instance of paraent class) info
# TODO: use chipID to config individual chip?
super().__init__()
......@@ -34,6 +34,8 @@ class Chip(FocalPlane):
self.flat_exptime = float(config["ins_effects"]['flat_exptime'])
self.readout_time = float(config["ins_effects"]['readout_time'])
self.logger = logger
# A chip ID must be assigned
self.chipID = int(chipID)
self._getChipRowCol()
......@@ -294,7 +296,7 @@ class Chip(FocalPlane):
fname = os.path.join(output_dir, h_prim['FILENAME']+'.fits')
hdu1.writeto(fname, output_verify='ignore', overwrite=True)
def addEffects(self, config, img, chip_output, filt, ra_cen, dec_cen, img_rot, exptime=150., pointing_ID=0, timestamp_obs=1621915200, pointing_type='MS', sky_map=None, tel=None):
def addEffects(self, config, img, chip_output, filt, ra_cen, dec_cen, img_rot, exptime=150., pointing_ID=0, timestamp_obs=1621915200, pointing_type='MS', sky_map=None, tel=None, logger=None):
SeedGainNonuni=int(config["random_seeds"]["seed_gainNonUniform"])
SeedBiasNonuni=int(config["random_seeds"]["seed_biasNonUniform"])
SeedRnNonuni = int(config["random_seeds"]["seed_rnNonUniform"])
......@@ -310,6 +312,7 @@ class Chip(FocalPlane):
BoolDeadPix = True
else:
BoolDeadPix = False
self.logger = logger
# Add sky background
if sky_map is None:
......@@ -324,8 +327,13 @@ class Chip(FocalPlane):
# Apply flat-field large scale structure for one chip
if config["ins_effects"]["flat_fielding"] == True:
print(" Creating and applying Flat-Fielding", flush=True)
print(img.bounds, flush=True)
if self.logger is not None:
self.logger.info(" Creating and applying Flat-Fielding")
msg = str(img.bounds)
self.logger.info(msg)
else:
print(" Creating and applying Flat-Fielding", flush=True)
print(img.bounds, flush=True)
flat_img = effects.MakeFlatSmooth(
img.bounds,
int(config["random_seeds"]["seed_flat"]))
......@@ -338,7 +346,10 @@ class Chip(FocalPlane):
# Apply Shutter-effect for one chip
if config["ins_effects"]["shutter_effect"] == True:
print(" Apply shutter effect", flush=True)
if self.logger is not None:
self.logger.info(" Apply shutter effect")
else:
print(" Apply shutter effect", flush=True)
shuttimg = effects.ShutterEffectArr(img, t_shutter=1.3, dist_bearing=735, dt=1E-3) # shutter effect normalized image for this chip
if self.survey_type == "photometric":
img *= shuttimg
......@@ -356,7 +367,10 @@ class Chip(FocalPlane):
# Add cosmic-rays
if config["ins_effects"]["cosmic_ray"] == True and pointing_type=='MS':
print(" Adding Cosmic-Ray", flush=True)
if self.logger is not None:
self.logger.info((" Adding Cosmic-Ray"))
else:
print(" Adding Cosmic-Ray", flush=True)
cr_map, cr_event_num = effects.produceCR_Map(
xLen=self.npix_x, yLen=self.npix_y,
exTime=self.exptime+0.5*self.readout_time,
......@@ -389,7 +403,10 @@ class Chip(FocalPlane):
# Apply PRNU effect and output PRNU flat file:
if config["ins_effects"]["prnu_effect"] == True:
print(" Applying PRNU effect", flush=True)
if self.logger is not None:
self.logger.info(" Applying PRNU effect")
else:
print(" Applying PRNU effect", flush=True)
prnu_img = effects.PRNU_Img(
xsize=self.npix_x,
ysize=self.npix_y,
......@@ -413,29 +430,42 @@ class Chip(FocalPlane):
# Apply Bad lines
if config["ins_effects"]["add_badcolumns"] == True:
img = effects.BadColumns(img, seed=SeedBadColumns, chipid=self.chipID)
img = effects.BadColumns(img, seed=SeedBadColumns, chipid=self.chipID, logger=self.logger)
# Add Bias level
if config["ins_effects"]["add_bias"] == True:
print(" Adding Bias level and 16-channel non-uniformity")
if self.logger is not None:
self.logger.info(" Adding Bias level and 16-channel non-uniformity")
else:
print(" Adding Bias level and 16-channel non-uniformity")
img = effects.AddBiasNonUniform16(img,
bias_level=float(config["ins_effects"]["bias_level"]),
nsecy = 2, nsecx=8,
seed=SeedBiasNonuni+self.chipID)
seed=SeedBiasNonuni+self.chipID,
logger=self.logger)
# Apply Nonlinearity on the chip image
if config["ins_effects"]["non_linear"] == True:
print(" Applying Non-Linearity on the chip image", flush=True)
if self.logger is not None:
self.logger.info(" Applying Non-Linearity on the chip image")
else:
print(" Applying Non-Linearity on the chip image", flush=True)
img = effects.NonLinearity(GSImage=img, beta1=5.e-7, beta2=0)
# Apply CCD Saturation & Blooming
if config["ins_effects"]["saturbloom"] == True:
print(" Applying CCD Saturation & Blooming")
if self.logger is not None:
self.logger.info(" Applying CCD Saturation & Blooming")
else:
print(" Applying CCD Saturation & Blooming")
img = effects.SaturBloom(GSImage=img, nsect_x=1, nsect_y=1, fullwell=fullwell)
# Apply CTE Effect
if config["ins_effects"]["cte_trail"] == True:
print(" Apply CTE Effect")
if self.logger is not None:
self.logger.info(" Apply CTE Effect")
else:
print(" Apply CTE Effect")
img = effects.CTE_Effect(GSImage=img, threshold=27)
# Add Read-out Noise
......@@ -447,11 +477,15 @@ class Chip(FocalPlane):
# Apply Gain & Quantization
print(" Applying Gain (and 16 channel non-uniformity) & Quantization", flush=True)
if self.logger is not None:
self.logger.info(" Applying Gain (and 16 channel non-uniformity) & Quantization")
else:
print(" Applying Gain (and 16 channel non-uniformity) & Quantization", flush=True)
img = effects.ApplyGainNonUniform16(
img, gain=self.gain,
nsecy = 2, nsecx=8,
seed=SeedGainNonuni+self.chipID)
seed=SeedGainNonuni+self.chipID,
logger=self.logger)
img.array[img.array > 65535] = 65535
img.replaceNegative(replace_value=0)
img.quantize()
......@@ -461,14 +495,18 @@ class Chip(FocalPlane):
######################################################################################
# Bias output
if config["output_setting"]["bias_output"] == True and pointing_type=='CAL':
print(" Output N frame Bias files", flush=True)
if self.logger is not None:
self.logger.info(" Output N frame Bias files")
else:
print(" Output N frame Bias files", flush=True)
NBias = int(config["ins_effects"]["NBias"])
for i in range(NBias):
BiasCombImg, BiasTag = effects.MakeBiasNcomb(
self.npix_x, self.npix_y,
bias_level=float(config["ins_effects"]["bias_level"]),
ncombine=1, read_noise=self.read_noise, gain=1,
seed=SeedBiasNonuni+self.chipID)
seed=SeedBiasNonuni+self.chipID,
logger=self.logger)
if config["ins_effects"]["cosmic_ray"] == True:
if config["ins_effects"]["cray_differ"] == True:
cr_map, cr_event_num = effects.produceCR_Map(
......@@ -484,16 +522,20 @@ class Chip(FocalPlane):
# Non-Linearity for Bias
if config["ins_effects"]["non_linear"] == True:
print(" Applying Non-Linearity on the Bias image", flush=True)
if self.logger is not None:
self.logger.info(" Applying Non-Linearity on the Bias image")
else:
print(" Applying Non-Linearity on the Bias image", flush=True)
BiasCombImg = effects.NonLinearity(GSImage=BiasCombImg, beta1=5.e-7, beta2=0)
# Apply Bad lines
if config["ins_effects"]["add_badcolumns"] == True:
BiasCombImg = effects.BadColumns(BiasCombImg-float(config["ins_effects"]["bias_level"])+5, seed=SeedBadColumns, chipid=self.chipID) + float(config["ins_effects"]["bias_level"])-5
BiasCombImg = effects.BadColumns(BiasCombImg-float(config["ins_effects"]["bias_level"])+5, seed=SeedBadColumns, chipid=self.chipID, logger=self.logger) + float(config["ins_effects"]["bias_level"])-5
BiasCombImg = effects.ApplyGainNonUniform16(BiasCombImg, gain=self.gain,
nsecy = 2, nsecx=8,
seed=SeedGainNonuni+self.chipID)
seed=SeedGainNonuni+self.chipID,
logger=self.logger)
# BiasCombImg = effects.AddOverscan(
# BiasCombImg,
# overscan=float(config["ins_effects"]["bias_level"])-2, gain=self.gain,
......@@ -521,7 +563,10 @@ class Chip(FocalPlane):
# Export combined (ncombine, Vignetting + PRNU) & single vignetting flat-field file
if config["output_setting"]["flat_output"] == True and pointing_type=='CAL':
print(" Output N frame Flat-Field files", flush=True)
if self.logger is not None:
self.logger.info(" Output N frame Flat-Field files")
else:
print(" Output N frame Flat-Field files", flush=True)
NFlat = int(config["ins_effects"]["NFlat"])
if config["ins_effects"]["add_bias"] == True:
biaslevel = self.bias_level
......@@ -539,7 +584,8 @@ class Chip(FocalPlane):
gain=1,
overscan=overscan,
biaslevel=0,
seed_bias=SeedDefective+self.chipID
seed_bias=SeedDefective+self.chipID,
logger=self.logger
)
if config["ins_effects"]["cosmic_ray"] == True:
if config["ins_effects"]["cray_differ"] == True:
......@@ -555,7 +601,10 @@ class Chip(FocalPlane):
del cr_map
if config["ins_effects"]["non_linear"] == True:
print(" Applying Non-Linearity on the Flat image", flush=True)
if self.logger is not None:
self.logger.info(" Applying Non-Linearity on the Flat image")
else:
print(" Applying Non-Linearity on the Flat image", flush=True)
FlatCombImg = effects.NonLinearity(GSImage=FlatCombImg, beta1=5.e-7, beta2=0)
if config["ins_effects"]["cte_trail"] == True:
......@@ -568,16 +617,20 @@ class Chip(FocalPlane):
# Apply Bad lines
if config["ins_effects"]["add_badcolumns"] == True:
FlatCombImg = effects.BadColumns(FlatCombImg, seed=SeedBadColumns, chipid=self.chipID)
FlatCombImg = effects.BadColumns(FlatCombImg, seed=SeedBadColumns, chipid=self.chipID, logger=self.logger)
# Add Bias level
if config["ins_effects"]["add_bias"] == True:
print(" Adding Bias level and 16-channel non-uniformity")
if self.logger is not None:
self.logger.info(" Adding Bias level and 16-channel non-uniformity")
else:
print(" Adding Bias level and 16-channel non-uniformity")
# img += float(config["ins_effects"]["bias_level"])
FlatCombImg = effects.AddBiasNonUniform16(FlatCombImg,
bias_level=biaslevel,
nsecy = 2, nsecx=8,
seed=SeedBiasNonuni+self.chipID)
seed=SeedBiasNonuni+self.chipID,
logger=self.logger)
# Add Read-out Noise
if config["ins_effects"]["add_readout"] == True:
......@@ -588,7 +641,8 @@ class Chip(FocalPlane):
FlatCombImg = effects.ApplyGainNonUniform16(FlatCombImg, gain=self.gain,
nsecy = 2, nsecx=8,
seed=SeedGainNonuni+self.chipID)
seed=SeedGainNonuni+self.chipID,
logger=self.logger)
# FlatCombImg = effects.AddOverscan(FlatCombImg, overscan=overscan, gain=self.gain, widthl=27, widthr=27, widtht=8, widthb=8)
FlatCombImg.replaceNegative(replace_value=0)
FlatCombImg.quantize()
......@@ -618,7 +672,10 @@ class Chip(FocalPlane):
# Export Dark current images
if config["output_setting"]["dark_output"] == True and pointing_type=='CAL':
print(" Output N frame Dark Current files", flush=True)
if self.logger is not None:
self.logger.info(" Output N frame Dark Current files")
else:
print(" Output N frame Dark Current files", flush=True)
NDark = int(config["ins_effects"]["NDark"])
if config["ins_effects"]["add_bias"] == True:
biaslevel = self.bias_level
......@@ -631,7 +688,8 @@ class Chip(FocalPlane):
self.npix_x, self.npix_y,
overscan=overscan, bias_level=0, darkpsec=0.02, exptime=self.dark_exptime+0.5*self.readout_time,
ncombine=1, read_noise=self.read_noise,
gain=1, seed_bias=SeedBiasNonuni+self.chipID)
gain=1, seed_bias=SeedBiasNonuni+self.chipID,
logger=self.logger)
if config["ins_effects"]["cosmic_ray"] == True:
if config["ins_effects"]["cray_differ"] == True:
cr_map, cr_event_num = effects.produceCR_Map(
......@@ -665,7 +723,10 @@ class Chip(FocalPlane):
# Non-Linearity for Dark
if config["ins_effects"]["non_linear"] == True:
print(" Applying Non-Linearity on the Dark image", flush=True)
if self.logger is not None:
self.logger.info(" Applying Non-Linearity on the Dark image")
else:
print(" Applying Non-Linearity on the Dark image", flush=True)
DarkCombImg = effects.NonLinearity(GSImage=DarkCombImg, beta1=5.e-7, beta2=0)
if config["ins_effects"]["cte_trail"] == True:
......@@ -678,16 +739,20 @@ class Chip(FocalPlane):
# Apply Bad lines
if config["ins_effects"]["add_badcolumns"] == True:
DarkCombImg = effects.BadColumns(DarkCombImg, seed=SeedBadColumns, chipid=self.chipID)
DarkCombImg = effects.BadColumns(DarkCombImg, seed=SeedBadColumns, chipid=self.chipID, logger=self.logger)
# Add Bias level
if config["ins_effects"]["add_bias"] == True:
print(" Adding Bias level and 16-channel non-uniformity")
if self.logger is not None:
self.logger.info(" Adding Bias level and 16-channel non-uniformity")
else:
print(" Adding Bias level and 16-channel non-uniformity")
# img += float(config["ins_effects"]["bias_level"])
DarkCombImg = effects.AddBiasNonUniform16(DarkCombImg,
bias_level=biaslevel,
nsecy = 2, nsecx=8,
seed=SeedBiasNonuni+self.chipID)
seed=SeedBiasNonuni+self.chipID,
logger=self.logger)
# Add Read-out Noise
if config["ins_effects"]["add_readout"] == True:
......@@ -699,7 +764,8 @@ class Chip(FocalPlane):
DarkCombImg = effects.ApplyGainNonUniform16(
DarkCombImg, gain=self.gain,
nsecy = 2, nsecx=8,
seed=SeedGainNonuni+self.chipID)
seed=SeedGainNonuni+self.chipID,
logger=self.logger)
# DarkCombImg = effects.AddOverscan(
# DarkCombImg,
# overscan=overscan, gain=self.gain,
......
ObservationSim/Instrument/Chip/Effects.py
View file @ 3a96ec26
......@@ -69,7 +69,7 @@ def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed=
return GSImage
def BadColumns(GSImage, seed=20240309, chipid=1):
def BadColumns(GSImage, seed=20240309, chipid=1, logger=None):
# Set bad column values
ysize,xsize = GSImage.array.shape
subarr = GSImage.array[int(ysize*0.1):int(ysize*0.12), int(xsize*0.1):int(xsize*0.12)]
......@@ -85,7 +85,10 @@ def BadColumns(GSImage, seed=20240309, chipid=1):
nbadsecA,nbadsecD = rgn.integers(low=1, high=5, size=2)
collen = rgcollen.integers(low=int(ysize*0.1), high=int(ysize*0.7), size=(nbadsecA+nbadsecD))
xposit = rgxpos.integers(low=int(xsize*0.05), high=int(xsize*0.95), size=(nbadsecA+nbadsecD))
print(xposit+1)
if logger is not None:
logger.info(xposit+1)
else:
print(xposit+1)
# signs = 2*rgdn.integers(0,2,size=(nbadsecA+nbadsecD))-1
# if meanimg>0:
dn = rgdn.integers(low=np.abs(meanimg)*1.3+50, high=np.abs(meanimg)*2+150, size=(nbadsecA+nbadsecD)) #*signs
......@@ -98,7 +101,7 @@ def BadColumns(GSImage, seed=20240309, chipid=1):
return GSImage
def AddBiasNonUniform16(GSImage, bias_level = 500, nsecy = 2, nsecx=8, seed=202102):
def AddBiasNonUniform16(GSImage, bias_level = 500, nsecy = 2, nsecx=8, seed=202102, logger=None):
# Generate Bias and its non-uniformity, and add the 16 bias values to the GS-Image
rg = Generator(PCG64(int(seed)))
Random16 = (rg.random(nsecy*nsecx)-0.5)*20
......@@ -106,7 +109,11 @@ def AddBiasNonUniform16(GSImage, bias_level = 500, nsecy = 2, nsecx=8, seed=2021
BiasLevel = np.zeros((nsecy,nsecx))
elif bias_level>0:
BiasLevel = Random16.reshape((nsecy,nsecx)) + bias_level
print(" Biases of 16 channels:\n",BiasLevel)
if logger is not None:
msg = str(" Biases of 16 channels: " + str(BiasLevel))
logger.info(msg)
else:
print(" Biases of 16 channels:\n",BiasLevel)
arrshape = GSImage.array.shape
secsize_x = int(arrshape[1]/nsecx)
secsize_y = int(arrshape[0]/nsecy)
......@@ -116,14 +123,15 @@ def AddBiasNonUniform16(GSImage, bias_level = 500, nsecy = 2, nsecx=8, seed=2021
return GSImage
def MakeBiasNcomb(npix_x, npix_y, bias_level=500, ncombine=1, read_noise=5, gain=1, seed=202102):
def MakeBiasNcomb(npix_x, npix_y, bias_level=500, ncombine=1, read_noise=5, gain=1, seed=202102, logger=None):
# Start with 0 value bias GS-Image
ncombine=int(ncombine)
BiasSngImg0 = galsim.Image(npix_x, npix_y, init_value=0)
BiasSngImg = AddBiasNonUniform16(BiasSngImg0,
bias_level=bias_level,
nsecy = 2, nsecx=8,
seed=int(seed))
seed=int(seed),
logger=logger)
BiasCombImg = BiasSngImg*ncombine
rng = galsim.UniformDeviate()
NoiseBias = galsim.GaussianNoise(rng=rng, sigma=read_noise*ncombine**0.5)
......@@ -139,12 +147,16 @@ def MakeBiasNcomb(npix_x, npix_y, bias_level=500, ncombine=1, read_noise=5, gain
return BiasCombImg, BiasTag
def ApplyGainNonUniform16(GSImage, gain=1, nsecy = 2, nsecx=8, seed=202102):
def ApplyGainNonUniform16(GSImage, gain=1, nsecy = 2, nsecx=8, seed=202102, logger=None):
# Generate Gain non-uniformity, and multipy the different factors (mean~1 with sigma~1%) to the GS-Image
rg = Generator(PCG64(int(seed)))
Random16 = (rg.random(nsecy*nsecx)-0.5)*0.04+1 # sigma~1%
Gain16 = Random16.reshape((nsecy,nsecx))/gain
print("Gain of 16 channels: ",Gain16)
if logger is not None:
msg = str("Gain of 16 channels: " + str(Gain16))
logger.info(msg)
else:
print("Gain of 16 channels: ",Gain16)
arrshape = GSImage.array.shape
secsize_x = int(arrshape[1]/nsecx)
secsize_y = int(arrshape[0]/nsecy)
......@@ -154,12 +166,16 @@ def ApplyGainNonUniform16(GSImage, gain=1, nsecy = 2, nsecx=8, seed=202102):
return GSImage
def GainsNonUniform16(GSImage, gain=1, nsecy = 2, nsecx=8, seed=202102):
def GainsNonUniform16(GSImage, gain=1, nsecy = 2, nsecx=8, seed=202102, logger=None):
# Generate Gain non-uniformity, and multipy the different factors (mean~1 with sigma~1%) to the GS-Image
rg = Generator(PCG64(int(seed)))
Random16 = (rg.random(nsecy*nsecx)-0.5)*0.04+1 # sigma~1%
Gain16 = Random16.reshape((nsecy,nsecx))/gain
print(seed-20210202, "Gains of 16 channels:\n", Gain16)
if logger is not None:
msg = str(seed-20210202, "Gains of 16 channels: " + str(Gain16))
logger.info(msg)
else:
print(seed-20210202, "Gains of 16 channels:\n", Gain16)
# arrshape = GSImage.array.shape
# secsize_x = int(arrshape[1]/nsecx)
# secsize_y = int(arrshape[0]/nsecy)
......@@ -186,7 +202,7 @@ def MakeFlatSmooth(GSBounds, seed):
return FlatImg
def MakeFlatNcomb(flat_single_image, ncombine=1, read_noise=5, gain=1, overscan=500, biaslevel=500, seed_bias=20210311):
def MakeFlatNcomb(flat_single_image, ncombine=1, read_noise=5, gain=1, overscan=500, biaslevel=500, seed_bias=20210311, logger=None):
ncombine=int(ncombine)
FlatCombImg = flat_single_image*ncombine
rng = galsim.UniformDeviate()
......@@ -200,7 +216,8 @@ def MakeFlatNcomb(flat_single_image, ncombine=1, read_noise=5, gain=1, overscan=
FlatCombImg,
bias_level=biaslevel,
nsecy=2, nsecx=8,
seed=seed_bias)
seed=seed_bias,
logger=logger)
if ncombine == 1:
FlatTag = 'Single'
pass
......@@ -212,7 +229,7 @@ def MakeFlatNcomb(flat_single_image, ncombine=1, read_noise=5, gain=1, overscan=
return FlatCombImg, FlatTag
def MakeDarkNcomb(npix_x, npix_y, overscan=500, bias_level=500, seed_bias=202102, darkpsec=0.02, exptime=150, ncombine=10, read_noise=5, gain=1):
def MakeDarkNcomb(npix_x, npix_y, overscan=500, bias_level=500, seed_bias=202102, darkpsec=0.02, exptime=150, ncombine=10, read_noise=5, gain=1, logger=None):
ncombine=int(ncombine)
darkpix = darkpsec*exptime
DarkSngImg = galsim.Image(npix_x, npix_y, init_value=darkpix)
......@@ -227,7 +244,8 @@ def MakeDarkNcomb(npix_x, npix_y, overscan=500, bias_level=500, seed_bias=202102
DarkCombImg,
bias_level=bias_level,
nsecy = 2, nsecx=8,
seed=int(seed_bias))
seed=int(seed_bias),
logger=logger)
if ncombine == 1:
DarkTag = 'Single'
pass
......
ObservationSim/MockObject/Galaxy.py
View file @ 3a96ec26
......@@ -10,8 +10,8 @@ from ObservationSim.MockObject.SpecDisperser import SpecDisperser
from ObservationSim.MockObject.MockObject import MockObject
class Galaxy(MockObject):
def __init__(self, param, rotation=None):
super().__init__(param)
def __init__(self, param, rotation=None, logger=None):
super().__init__(param, logger=logger)
self.thetaR = self.param["theta"]
self.bfrac = self.param["bfrac"]
self.hlr_disk = self.param["hlr_disk"]
......
ObservationSim/MockObject/MockObject.py
View file @ 3a96ec26
......@@ -8,7 +8,7 @@ from ObservationSim.MockObject._util import integrate_sed_bandpass, getNormFacto
from ObservationSim.MockObject.SpecDisperser import SpecDisperser
class MockObject(object):
def __init__(self, param):
def __init__(self, param, logger=None):
self.param = param
if self.param["star"] == 0:
......@@ -45,6 +45,8 @@ class MockObject(object):
self.e1_disk, self.e2_disk = 0., 0.
self.e1_bulge, self.e2_bulge = 0., 0.
self.logger = logger
def getMagFilter(self, filt):
if filt.filter_type in ["GI", "GV", "GU"]:
return self.param["mag_use_normal"]
......
ObservationSim/MockObject/Quasar.py
View file @ 3a96ec26
......@@ -9,8 +9,8 @@ from ObservationSim.MockObject.MockObject import MockObject
from ObservationSim.MockObject._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG
class Quasar(MockObject):
def __init__(self, param):
super().__init__(param)
def __init__(self, param, logger=None):
super().__init__(param, logger=logger)
def load_SED(self, survey_type, sed_path=None, cosids=None, objtypes=None, sed_templates=None, normFilter=None, target_filt=None):
if survey_type == "photometric":
......
ObservationSim/MockObject/Star.py
View file @ 3a96ec26
......@@ -9,8 +9,8 @@ from ObservationSim.MockObject._util import integrate_sed_bandpass, getNormFacto
from ObservationSim.MockObject.MockObject import MockObject
class Star(MockObject):
def __init__(self, param):
super().__init__(param)
def __init__(self, param, logger=None):
super().__init__(param, logger=logger)
def unload_SED(self):
"""(Test) free up SED memory
......
ObservationSim/ObservationSim.py
View file @ 3a96ec26
......@@ -14,6 +14,7 @@ from ObservationSim.Instrument.Chip import Effects
from ObservationSim.MockObject import calculateSkyMap_split_g
from ObservationSim.PSF import PSFGauss, FieldDistortion, PSFInterp
from ObservationSim._util import get_shear_field, makeSubDir_PointingList
from ObservationSim.Astrometry.Astrometry_util import on_orbit_obs_position
class Observation(object):
def __init__(self, config, Catalog, work_dir=None, data_dir=None):
......@@ -56,30 +57,68 @@ class Observation(object):
def run_one_chip(self, chip, filt, pointing, chip_output, wcs_fp=None, psf_model=None, shear_cat_file=None, cat_dir=None, sed_dir=None):
print(':::::::::::::::::::Current Pointing Information::::::::::::::::::')
print("RA: %f, DEC; %f" % (pointing.ra, pointing.dec))
print("Time: %s" % datetime.fromtimestamp(pointing.timestamp).isoformat())
print("Exposure time: %f" % pointing.exp_time)
print("Satellite Position (x, y, z): (%f, %f, %f)" % (pointing.sat_x, pointing.sat_y, pointing.sat_z))
print("Satellite Velocity (x, y, z): (%f, %f, %f)" % (pointing.sat_vx, pointing.sat_vy, pointing.sat_vz))
print("Position Angle: %f" % pointing.img_pa.deg)
print('Chip : %d' % chip.chipID)
print(':::::::::::::::::::::::::::END:::::::::::::::::::::::::::::::::::')
# print(':::::::::::::::::::Current Pointing Information::::::::::::::::::')
# print("RA: %f, DEC; %f" % (pointing.ra, pointing.dec))
# print("Time: %s" % datetime.fromtimestamp(pointing.timestamp).isoformat())
# print("Exposure time: %f" % pointing.exp_time)
# print("Satellite Position (x, y, z): (%f, %f, %f)" % (pointing.sat_x, pointing.sat_y, pointing.sat_z))
# print("Satellite Velocity (x, y, z): (%f, %f, %f)" % (pointing.sat_vx, pointing.sat_vy, pointing.sat_vz))
# print("Position Angle: %f" % pointing.img_pa.deg)
# print('Chip : %d' % chip.chipID)
# print(':::::::::::::::::::::::::::END:::::::::::::::::::::::::::::::::::')
chip_output.logger.info(':::::::::::::::::::Current Pointing Information::::::::::::::::::')
chip_output.logger.info("RA: %f, DEC; %f" % (pointing.ra, pointing.dec))
chip_output.logger.info("Time: %s" % datetime.fromtimestamp(pointing.timestamp).isoformat())
chip_output.logger.info("Exposure time: %f" % pointing.exp_time)
chip_output.logger.info("Satellite Position (x, y, z): (%f, %f, %f)" % (pointing.sat_x, pointing.sat_y, pointing.sat_z))
chip_output.logger.info("Satellite Velocity (x, y, z): (%f, %f, %f)" % (pointing.sat_vx, pointing.sat_vy, pointing.sat_vz))
chip_output.logger.info("Position Angle: %f" % pointing.img_pa.deg)
chip_output.logger.info('Chip : %d' % chip.chipID)
chip_output.logger.info(':::::::::::::::::::::::::::END:::::::::::::::::::::::::::::::::::')
if self.config["psf_setting"]["psf_model"] == "Gauss":
psf_model = PSFGauss(chip=chip)
elif self.config["psf_setting"]["psf_model"] == "Interp":
psf_model = PSFInterp(chip=chip, PSF_data_file=self.path_dict["psf_dir"])
else:
print("unrecognized PSF model type!!", flush=True)
# print("unrecognized PSF model type!!", flush=True)
chip_output.logger.error("unrecognized PSF model type!!", flush=True)
# Get (extra) shear fields
if shear_cat_file is not None:
self.g1_field, self.g2_field, self.nshear = get_shear_field(config=self.config, shear_cat_file=shear_cat_file)
# Apply astrometric simulation for pointing
if self.config["obs_setting"]["enable_astrometric_model"]:
dt = datetime.fromtimestamp(pointing.timestamp)
date_str = dt.date().isoformat()
time_str = dt.time().isoformat()
ra_cen, dec_cen = on_orbit_obs_position(
input_ra_list=[pointing.ra],
input_dec_list=[pointing.dec],
input_pmra_list=[0.],
input_pmdec_list=[0.],
input_rv_list=[0.],
input_parallax_list=[1e-9],
input_nstars=1,
input_x=pointing.sat_x,
input_y=pointing.sat_y,
input_z=pointing.sat_z,
input_vx=pointing.sat_vx,
input_vy=pointing.sat_vy,
input_vz=pointing.sat_vz,
input_epoch="J2015.5",
input_date_str=date_str,
input_time_str=time_str
)
ra_cen, dec_cen = ra_cen[0], dec_cen[0]
else:
ra_cen = pointing.ra
dec_cen = pointing.dec
# Get WCS for the focal plane
if wcs_fp == None:
wcs_fp = self.focal_plane.getTanWCS(pointing.ra, pointing.dec, pointing.img_pa, chip.pix_scale)
wcs_fp = self.focal_plane.getTanWCS(ra_cen, dec_cen, pointing.img_pa, chip.pix_scale)
# Create chip Image
chip.img = galsim.ImageF(chip.npix_x, chip.npix_y)
......@@ -92,14 +131,18 @@ class Observation(object):
elif chip.survey_type == "spectroscopic":
flat_normal = np.ones_like(chip.img.array)
if self.config["ins_effects"]["flat_fielding"] == True:
print("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding"%chip.chipID, flush=True)
print(chip.img.bounds, flush=True)
# print("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding"%chip.chipID, flush=True)
# print(chip.img.bounds, flush=True)
chip_output.logger.info("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding"%chip.chipID)
msg = str(chip.img.bounds)
chip_output.logger.info(msg)
flat_img = Effects.MakeFlatSmooth(
chip.img.bounds,
int(self.config["random_seeds"]["seed_flat"]))
flat_normal = flat_normal * flat_img.array / np.mean(flat_img.array)
if self.config["ins_effects"]["shutter_effect"] == True:
print("SLS flat preprocess,CHIP %d : Apply shutter effect"%chip.chipID, flush=True)
# print("SLS flat preprocess,CHIP %d : Apply shutter effect"%chip.chipID, flush=True)
chip_output.logger.info("SLS flat preprocess,CHIP %d : Apply shutter effect"%chip.chipID)
shuttimg = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735,
dt=1E-3) # shutter effect normalized image for this chip
flat_normal = flat_normal*shuttimg
......@@ -115,7 +158,7 @@ class Observation(object):
if pointing.pointing_type == 'MS':
# Load catalogues and templates
self.cat = self.Catalog(config=self.config, chip=chip, pointing=pointing, cat_dir=cat_dir, sed_dir=sed_dir)
self.cat = self.Catalog(config=self.config, chip=chip, pointing=pointing, cat_dir=cat_dir, sed_dir=sed_dir, logger=chip_output.logger)
self.nobj = len(self.cat.objs)
# Loop over objects
......@@ -123,8 +166,11 @@ class Observation(object):
bright_obj = 0
dim_obj = 0
for j in range(self.nobj):
# (DEBUG)
# if j >= 10:
# break
obj = self.cat.objs[j]
if obj.type == 'star' and self.config["run_option"]["galaxy_only"]:
continue
......@@ -144,7 +190,8 @@ class Observation(object):
norm_filt=norm_filt,
)
except Exception as e:
print(e)
# print(e)
chip_output.logger.error(e)
continue
# Exclude very bright/dim objects (for now)
......@@ -171,11 +218,13 @@ class Observation(object):
# TODO: every object with individual shear from input catalog(s)
obj.g1, obj.g2 = self.g1_field[j], self.g2_field[j]
except:
print("failed to load external shear.")
# print("failed to load external shear.")
chip_output.logger.error("failed to load external shear.")
pass
elif self.config["shear_setting"]["shear_type"] == "catalog":
pass
else:
chip_output.logger.error("Unknown shear input")
raise ValueError("Unknown shear input")
pos_img, offset, local_wcs = obj.getPosImg_Offset_WCS(img=chip.img, fdmodel=self.fd_model, chip=chip, verbose=False)
......@@ -224,7 +273,8 @@ class Observation(object):
# print("object omitted", flush=True)
continue
except Exception as e:
print(e)
# print(e)
chip_output.logger.error(e)
pass
# Unload SED:
obj.unload_SED()
......@@ -233,7 +283,8 @@ class Observation(object):
del psf_model
del self.cat
print("check running:1: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True)
# print("check running:1: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True)
chip_output.logger.info("check running:1: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
# Detector Effects
# ===========================================================
......@@ -249,7 +300,8 @@ class Observation(object):
pointing_ID=pointing.id,
timestamp_obs=pointing.timestamp,
pointing_type=pointing.pointing_type,
sky_map=sky_map, tel = self.tel)
sky_map=sky_map, tel = self.tel,
logger=chip_output.logger)
if pointing.pointing_type == 'MS':
datetime_obs = datetime.fromtimestamp(pointing.timestamp)
......@@ -290,12 +342,16 @@ class Observation(object):
hdu1 = fits.HDUList([hdu1, hdu2])
fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits')
hdu1.writeto(fname, output_verify='ignore', overwrite=True)
print("# objects that are too bright %d out of %d"%(bright_obj, self.nobj))
print("# objects that are too dim %d out of %d"%(dim_obj, self.nobj))
print("# objects that are missed %d out of %d"%(missed_obj, self.nobj))
# print("# objects that are too bright %d out of %d"%(bright_obj, self.nobj))
# print("# objects that are too dim %d out of %d"%(dim_obj, self.nobj))
# print("# objects that are missed %d out of %d"%(missed_obj, self.nobj))
chip_output.logger.info("# objects that are too bright %d out of %d"%(bright_obj, self.nobj))
chip_output.logger.info("# objects that are too dim %d out of %d"%(dim_obj, self.nobj))
chip_output.logger.info("# objects that are missed %d out of %d"%(missed_obj, self.nobj))
del chip.img
print("check running:2: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True)
# print("check running:2: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True)
chip_output.logger.info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ))
def runExposure_MPI_PointingList(self, pointing_list, shear_cat_file=None, chips=None, use_mpi=False):
if use_mpi:
......@@ -334,7 +390,7 @@ class Observation(object):
chip = run_chips[ichip]
filt = run_filts[ichip]
print("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid), flush=True)
# print("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid), flush=True)
chip_output = ChipOutput(
config=self.config,
focal_plane=self.focal_plane,
......@@ -345,6 +401,7 @@ class Observation(object):
pointing_ID=pointing_ID,
subdir=sub_img_dir,
prefix=prefix)
chip_output.logger.info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid))
self.run_one_chip(
chip=chip,
filt=filt,
......@@ -352,3 +409,6 @@ class Observation(object):
pointing=pointing,
cat_dir=self.path_dict["cat_dir"])
print("finished running chip#%d..."%(chip.chipID), flush=True)
chip_output.logger.info("finished running chip#%d..."%(chip.chipID))
for handler in chip_output.logger.handlers[:]:
chip_output.logger.removeHandler(handler)
ObservationSim/_util.py
View file @ 3a96ec26
......@@ -104,6 +104,7 @@ def make_run_dirs(work_dir, run_name, pointing_list):
os.makedirs(subImgDir, exist_ok=True)
except OSError:
pass
return imgDir
def imgName(tt=0):
ut = datetime.utcnow()
......
config/config_NGP.yaml
View file @ 3a96ec26
......@@ -12,14 +12,14 @@
# work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/"
work_dir: "/public/home/fangyuedong/temp/CSST/workplace/"
data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
run_name: "NGP_20220327"
run_name: "NGP_test"
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: null
pointing_file: null
pointing_dir: "/data/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_test_NGP_3.dat"
# Whether to use MPI
run_option:
......@@ -34,7 +34,7 @@ run_option:
out_cat_only: NO
# Only simulate stars?
star_only: NO
star_only: YES
# Only simulate galaxies?
galaxy_only: NO
......@@ -48,7 +48,7 @@ obs_setting:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "All": simulate full focal plane
survey_type: "All"
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
......@@ -73,7 +73,7 @@ obs_setting:
# - give a list of indexes of pointings: [ip_1, ip_2...]
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: null
run_pointings: [1]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
......@@ -92,7 +92,7 @@ obs_setting:
# Default path settings for WIDE survey simulation
input_path:
cat_dir: "OnOrbitCalibration/CTargets20211231"
star_cat: "CT-NGP_V2.2.hdf5"
star_cat: "CT-NGP_r1.8_G28.hdf5"
galaxy_cat: "galaxyCats_r_10.0_healpix_shift_192.859500_27.128300.hdf5"
SED_templates_path:
......
run_sim.py
View file @ 3a96ec26
......@@ -2,6 +2,7 @@ from ObservationSim.ObservationSim import Observation
from ObservationSim._util import parse_args, make_run_dirs, generate_pointing_list
import os
import yaml
import shutil
import gc
gc.enable()
......@@ -47,7 +48,8 @@ def run_sim(Catalog):
pointing_list = generate_pointing_list(config=config, pointing_filename=config['pointing_file'], data_dir=config['pointing_dir'])
# Make the main output directories
make_run_dirs(work_dir=config['work_dir'], run_name=config['run_name'], pointing_list=pointing_list)
run_dir = make_run_dirs(work_dir=config['work_dir'], run_name=config['run_name'], pointing_list=pointing_list)
shutil.copy(args.config_file, run_dir)
# Initialize the simulation
obs = Observation(config=config, Catalog=Catalog, work_dir=config['work_dir'], data_dir=config['data_dir'])
......@@ -63,6 +65,10 @@ if __name__=='__main__':
# from Catalog.Catalog_example import Catalog_example
# run_sim(Catalog=Catalog_example)
# To run cycle-3 simulation
from Catalog.C3Catalog import C3Catalog
run_sim(Catalog=C3Catalog)
# # To run cycle-3 simulation
# from Catalog.C3Catalog import C3Catalog
# run_sim(Catalog=C3Catalog)
# To run calibration field NGP simulation
from Catalog.NGPCatalog import NGPCatalog
run_sim(Catalog=NGPCatalog)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment