Commit d5059d30 authored by Fang Yuedong's avatar Fang Yuedong
Browse files

fixed photometry/bad column issue, modify detector effects order

parent ef8c4609
......@@ -37,8 +37,10 @@ class ChipOutput(object):
fmt5 = "%10s %8.4f %8.4f %8.4f\n"
self.hdr = hdr1 + hdr2 + hdr3 + hdr4 + hdr5
self.fmt = fmt1 + fmt2 + fmt3 + fmt4 + fmt5
print("pointing_type = %s\n"%(pointing_type))
if pointing_type == 'MS':
self.cat = open(os.path.join(self.subdir, self.cat_name), "w")
print("Creating catalog file %s ...\n"%(os.path.join(self.subdir, self.cat_name)))
self.cat.write(self.hdr)
def updateHDR(self, hdr):
......
This diff is collapsed.
......@@ -5,6 +5,7 @@ from numpy.core.fromnumeric import mean, size
from numpy.random import Generator, PCG64
import math
from numba import jit
from astropy import stats
def AddOverscan(GSImage, overscan=1000, gain=1, widthl=27, widthr=27, widtht=8, widthb=8, read_noise=5):
......@@ -31,7 +32,7 @@ def AddOverscan(GSImage, overscan=1000, gain=1, widthl=27, widthr=27, widtht=8,
return newimg
def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed=20210304, biaslevel=500):
def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed=20210304, biaslevel=0):
# Also called bad pixels, including hot pixels and dead pixels
# Hot Pixel > 20e-/s
# Dead Pixel < 70%*Mean
......@@ -68,10 +69,11 @@ def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed=
return GSImage
def BadColumns(GSImage, seed=20210309, chipid=1):
def BadColumns(GSImage, seed=20240309, chipid=1):
# Set bad column values
ysize,xsize = GSImage.array.shape
subarr = GSImage.array[int(ysize*0.1):int(ysize*0.12), int(xsize*0.1):int(xsize*0.12)]
subarr = stats.sigma_clip(subarr, sigma=4, cenfunc='median', maxiters=3, masked=False)
meanimg = np.median(subarr)
stdimg = np.std(subarr)
seed += chipid
......@@ -80,15 +82,19 @@ def BadColumns(GSImage, seed=20210309, chipid=1):
rgxpos = Generator(PCG64(int(seed*1.2)))
rgdn = Generator(PCG64(int(seed*1.3)))
nbadsecA,nbadsecD = rgn.integers(low=1, high=5, size=2)
nbadsecA,nbadsecD = rgn.integers(low=1, high=5, size=2)
collen = rgcollen.integers(low=int(ysize*0.1), high=int(ysize*0.7), size=(nbadsecA+nbadsecD))
xposit = rgxpos.integers(low=int(xsize*0.05), high=int(xsize*0.95), size=(nbadsecA+nbadsecD))
signs = 2*rgdn.integers(0,2,size=(nbadsecA+nbadsecD))-1
dn = rgdn.integers(low=meanimg*0.4, high=meanimg*0.8, size=(nbadsecA+nbadsecD))*signs
print(xposit+1)
# signs = 2*rgdn.integers(0,2,size=(nbadsecA+nbadsecD))-1
if meanimg>0:
dn = rgdn.integers(low=meanimg*1.3+50, high=meanimg*2+150, size=(nbadsecA+nbadsecD)) #*signs
elif meanimg<0:
dn = rgdn.integers(low=meanimg*2-150, high=meanimg*1.3-50, size=(nbadsecA+nbadsecD)) #*signs
for badcoli in range(nbadsecA):
GSImage.array[(ysize-collen[badcoli]):ysize,xposit[badcoli]:(xposit[badcoli]+1)] += np.random.random((collen[badcoli],1))*stdimg*3+dn[badcoli]
GSImage.array[(ysize-collen[badcoli]):ysize,xposit[badcoli]:(xposit[badcoli]+1)] = (np.abs(np.random.normal(0, stdimg*2, (collen[badcoli],1)))+dn[badcoli])
for badcoli in range(nbadsecD):
GSImage.array[0:collen[badcoli+nbadsecA],xposit[badcoli+nbadsecA]:(xposit[badcoli+nbadsecA]+1)] += np.random.random((collen[badcoli+nbadsecA],1))*stdimg*3+dn[badcoli+nbadsecA]
GSImage.array[0:collen[badcoli+nbadsecA],xposit[badcoli+nbadsecA]:(xposit[badcoli+nbadsecA]+1)] = (np.abs(np.random.normal(0, stdimg*2, (collen[badcoli+nbadsecA],1)))+dn[badcoli+nbadsecA])
return GSImage
......@@ -223,7 +229,7 @@ def PRNU_Img(xsize, ysize, sigma=0.01, seed=202101):
return prnuimg
def NonLinearity(GSImage, beta1=1E-7, beta2=1E-10):
def NonLinearity(GSImage, beta1=5E-7, beta2=0):
NonLinear_f = lambda x, beta_1, beta_2: x - beta_1*x*x + beta_2*x*x*x
GSImage.applyNonlinearity(NonLinear_f, beta1, beta2)
return GSImage
......
......@@ -73,5 +73,5 @@ class Filter(object):
def getPhotonE(self):
return photonEnergy(self.effective_wavelength)
def getSkyNoise(self, exptime, gain):
def getSkyNoise(self, exptime, gain=1.):
return self.sky_background * exptime / gain
......@@ -97,6 +97,7 @@ class Observation(object):
# Load SED
if obj.type == 'star':
normF = chip.normF_star
#continue
try:
obj.load_SED(
survey_type=chip.survey_type,
......@@ -108,6 +109,7 @@ class Observation(object):
continue
elif obj.type == 'galaxy': # or obj.type == quasar
normF = chip.normF_galaxy
# continue
obj.load_SED(
sed_path=sed_dir,
survey_type=chip.survey_type,
......@@ -116,6 +118,7 @@ class Observation(object):
target_filt=filt)
elif obj.type == 'quasar':
normF = chip.normF_galaxy
# continue
obj.load_SED(
sed_path=sed_dir,
survey_type=chip.survey_type,
......@@ -200,7 +203,7 @@ class Observation(object):
# Detector Effects
# ===========================================================
chip.img = chip.addNoise(config=self.config, tel=self.tel, filt=filt, img=chip.img, sky_map=sky_map)
# chip.img = chip.addNoise(config=self.config, tel=self.tel, filt=filt, img=chip.img, sky_map=sky_map)
# chip.img = chip.addEffects(config=self.config, img=chip.img, chip_output=chip_output, filt=filt, pointing_ID=pointing_ID)
# whether to output zero, dark, flat calibration images.
......@@ -214,7 +217,8 @@ class Observation(object):
img_rot=img_rot,
pointing_ID=pointing_ID,
timestamp_obs=timestamp_obs,
pointing_type=pointing_type)
pointing_type=pointing_type,
sky_map=sky_map)
if pointing_type == 'MS':
datetime_obs = datetime.fromtimestamp(timestamp_obs)
......@@ -309,13 +313,20 @@ class Observation(object):
sed_dir=self.path_dict["SED_dir"])
print("finished running chip#%d..."%(chip.chipID), flush=True)
def runExposure_MPI_PointingList(self, ra_cen=None, dec_cen=None, pRange=None, timestamp_obs=[1621915200], pointing_type=['MS'], img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None):
def runExposure_MPI_PointingList(self, ra_cen=None, dec_cen=None, pRange=None, timestamp_obs=np.array([1621915200]), pointing_type=np.array(['MS']), img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None):
comm = MPI.COMM_WORLD
ind_thread = comm.Get_rank()
num_thread = comm.Get_size()
nchips_per_fp = len(self.chip_list)
# TEMP
if len(timestamp_obs) == 1:
timestamp_obs = np.tile(timestamp_obs, len(ra_cen))
pointing_type = np.tile(pointing_type, len(ra_cen))
timestamp_obs = timestamp_obs[pRange]
pointing_type = pointing_type[pRange]
ra_cen = ra_cen[pRange]
dec_cen = dec_cen[pRange]
......@@ -325,11 +336,6 @@ class Observation(object):
else:
pStart = 0
# TEMP
if len(timestamp_obs) == 1:
timestamp_obs *= len(pRange)
pointing_type *= len(pRange)
for ipoint in range(len(ra_cen)):
for ichip in range(nchips_per_fp):
i = ipoint*nchips_per_fp + ichip
......
......@@ -171,8 +171,8 @@ class PSFInterp(PSFModel):
img = galsim.ImageF(imPSF, scale=pixSize)
gsp = galsim.GSParams(folding_threshold=folding_threshold)
self.psf = galsim.InterpolatedImage(img, gsparams=gsp)
gaussPSF = galsim.Gaussian(sigma=0.04, gsparams=gsp)
self.psf = galsim.Convolve(gaussPSF, self.psf)
# gaussPSF = galsim.Gaussian(sigma=0.04, gsparams=gsp)
# self.psf = galsim.Convolve(gaussPSF, self.psf)
return self.PSFspin(x=px/0.01, y=py/0.01)
return imPSF
......
......@@ -9,6 +9,8 @@ data_dir = "/data/simudata/CSSOSDataProductsSims/data/"
config_file = os.path.join(work_dir, "ObservationSim.cfg")
config = ReadConfig(config_file)
exp_time = float(config['exp_time'])
# Read Pointing list
pointing_file = os.path.join(data_dir, "pointing10_20210202.dat")
f = open(pointing_file, 'r')
......@@ -37,12 +39,16 @@ pointing_type = ['CAL']*ncal + pointing_type
t0 = datetime(2021, 5, 25, 12, 0, 0)
t = datetime.timestamp(t0)
timestamp_obs = []
delta_t = 10 # Time elapsed between exposures (minutes)
for i in range(len(pointing_type)):
timestamp_obs.append(t)
if pointing_type[i] == 'CAL':
t += 3 * 5 * 60 # 3 calibration exposure
t += 3 * delta_t * 60 # 3 calibration exposure
elif pointing_type[i] == 'MS':
t += 5 * 60
t += delta_t * 60
timestamp_obs = np.array(timestamp_obs)
pointing_type = np.array(pointing_type)
# Define the range of pointing list
pRange = range(0, 2)
\ No newline at end of file
......@@ -7,7 +7,7 @@
##mpdboot -n 10 -f ./mpd.hosts
##PBS -j oe
#PBS -l nodes=comput110:ppn=80
#PBS -l nodes=comput108:ppn=80
#####PBS -q longq
#PBS -q batch
#PBS -u fangyuedong
......
......@@ -20,4 +20,4 @@ gc.enable()
# Testing run pointing list (using MPI)
obs = Observation(work_dir=work_dir, data_dir=data_dir)
# obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type)
obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=150.)
obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=exp_time)
......@@ -7,6 +7,7 @@
# Observation setting
date_obs 210525 # Observation date [yymmdd]
time_obs 120000 # Observation time [hhmmss]
exp_time 150. # Exposure time [seconds]
ra_center 60.0 # Telesscope pointing center [degree] (Default)
dec_center -40.0
psf_rcont 0.15,0.8 # Radius of the 80% flux concentration (for a Gaussian PSF)
......@@ -18,7 +19,7 @@ survey_type Photometric # Survey simulation option:
# 'Photometric': only run sims for photometric chips
# 'Spectroscopic': only run sims for spectroscopic chps
# 'All': run sims for all chips
psf_model Interp # which PSF model to use:
psf_model Interp # Which PSF model to use:
# 'Gauss': simple gaussin profile
# 'Interp': Interpolated PSF from measured data
......@@ -26,8 +27,9 @@ psf_model Interp # which PSF model to use:
# Instrument effects
#===========================================================
field_dist y # Y/N: Whether to add field distortion
abs_back y # Y: add sky background + sky noise & dark + dark noise
# N: only add sky noise and dark noise
add_back y # Y/N: Whether to add sky background
add_dark y # Y/N: Whether to add dark noise
add_readout y # Y/N: Whether to add read-out (Gaussian) noise
add_bias y # Y/N: Whether to add bias-level to image
shutter_effect y # Y/N: Whether to apply shutter effect
shutter_output n # Y/N: Whether to export shutter effect 16-bit image (<=65535)
......@@ -56,6 +58,7 @@ NDark 1 # Number of frames to be exported
NFlat 1 # Number of frames to be exported
dark_exptime 300 # Exposure time for dark current frame
flat_exptime 150 # Exposure time for flat field frame
readout_time 40 # Time for read-out by each channel
#===========================================================
# Shear method
......@@ -85,6 +88,7 @@ seed_mock 12345678 # Seed for generating random values of ra,
seed_star 121212121 # Seed for generating random redshift, sed_type for stars
seed_gal 212121212 # Seed for generating random redshift, sed_type for galaxies
seed_Av 121212 # Seed for generating random intrinsic extinction
seed_poisson 20210601 # Seed for Poisson noise
seed_CR 20210317 # Seed for generating random cosmic ray map
seed_flat 20210101 # Seed for generating random flat field
seed_prnu 20210102 # Seed for photo-response non-uniformity
......@@ -93,4 +97,5 @@ seed_gainNonUniform 20210202 # Seed for Gain nonuniformity
seed_biasNonUniform 20210203 # Seed for Bias nonuniformity
seed_rnNonUniform 20210204 # Seed for ReadNoise nonuniformity
seed_badcolumns 20240309 # Initial Seed for Bad Columns
seed_defective 20210304 # Seed for Defective(bad) pixels
\ No newline at end of file
seed_defective 20210304 # Seed for Defective(bad) pixels
seed_readout 20210601 # Seed for Read-out gaussian noise
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment