Commit d5059d30 authored by Fang Yuedong's avatar Fang Yuedong
Browse files

fixed photometry/bad column issue, modify detector effects order

parent ef8c4609
...@@ -37,8 +37,10 @@ class ChipOutput(object): ...@@ -37,8 +37,10 @@ class ChipOutput(object):
fmt5 = "%10s %8.4f %8.4f %8.4f\n" fmt5 = "%10s %8.4f %8.4f %8.4f\n"
self.hdr = hdr1 + hdr2 + hdr3 + hdr4 + hdr5 self.hdr = hdr1 + hdr2 + hdr3 + hdr4 + hdr5
self.fmt = fmt1 + fmt2 + fmt3 + fmt4 + fmt5 self.fmt = fmt1 + fmt2 + fmt3 + fmt4 + fmt5
print("pointing_type = %s\n"%(pointing_type))
if pointing_type == 'MS': if pointing_type == 'MS':
self.cat = open(os.path.join(self.subdir, self.cat_name), "w") self.cat = open(os.path.join(self.subdir, self.cat_name), "w")
print("Creating catalog file %s ...\n"%(os.path.join(self.subdir, self.cat_name)))
self.cat.write(self.hdr) self.cat.write(self.hdr)
def updateHDR(self, hdr): def updateHDR(self, hdr):
......
This diff is collapsed.
...@@ -5,6 +5,7 @@ from numpy.core.fromnumeric import mean, size ...@@ -5,6 +5,7 @@ from numpy.core.fromnumeric import mean, size
from numpy.random import Generator, PCG64 from numpy.random import Generator, PCG64
import math import math
from numba import jit from numba import jit
from astropy import stats
def AddOverscan(GSImage, overscan=1000, gain=1, widthl=27, widthr=27, widtht=8, widthb=8, read_noise=5): def AddOverscan(GSImage, overscan=1000, gain=1, widthl=27, widthr=27, widtht=8, widthb=8, read_noise=5):
...@@ -31,7 +32,7 @@ def AddOverscan(GSImage, overscan=1000, gain=1, widthl=27, widthr=27, widtht=8, ...@@ -31,7 +32,7 @@ def AddOverscan(GSImage, overscan=1000, gain=1, widthl=27, widthr=27, widtht=8,
return newimg return newimg
def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed=20210304, biaslevel=500): def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed=20210304, biaslevel=0):
# Also called bad pixels, including hot pixels and dead pixels # Also called bad pixels, including hot pixels and dead pixels
# Hot Pixel > 20e-/s # Hot Pixel > 20e-/s
# Dead Pixel < 70%*Mean # Dead Pixel < 70%*Mean
...@@ -68,10 +69,11 @@ def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed= ...@@ -68,10 +69,11 @@ def DefectivePixels(GSImage, IfHotPix=True, IfDeadPix=True, fraction=1E-4, seed=
return GSImage return GSImage
def BadColumns(GSImage, seed=20210309, chipid=1): def BadColumns(GSImage, seed=20240309, chipid=1):
# Set bad column values # Set bad column values
ysize,xsize = GSImage.array.shape ysize,xsize = GSImage.array.shape
subarr = GSImage.array[int(ysize*0.1):int(ysize*0.12), int(xsize*0.1):int(xsize*0.12)] subarr = GSImage.array[int(ysize*0.1):int(ysize*0.12), int(xsize*0.1):int(xsize*0.12)]
subarr = stats.sigma_clip(subarr, sigma=4, cenfunc='median', maxiters=3, masked=False)
meanimg = np.median(subarr) meanimg = np.median(subarr)
stdimg = np.std(subarr) stdimg = np.std(subarr)
seed += chipid seed += chipid
...@@ -83,12 +85,16 @@ def BadColumns(GSImage, seed=20210309, chipid=1): ...@@ -83,12 +85,16 @@ def BadColumns(GSImage, seed=20210309, chipid=1):
nbadsecA,nbadsecD = rgn.integers(low=1, high=5, size=2) nbadsecA,nbadsecD = rgn.integers(low=1, high=5, size=2)
collen = rgcollen.integers(low=int(ysize*0.1), high=int(ysize*0.7), size=(nbadsecA+nbadsecD)) collen = rgcollen.integers(low=int(ysize*0.1), high=int(ysize*0.7), size=(nbadsecA+nbadsecD))
xposit = rgxpos.integers(low=int(xsize*0.05), high=int(xsize*0.95), size=(nbadsecA+nbadsecD)) xposit = rgxpos.integers(low=int(xsize*0.05), high=int(xsize*0.95), size=(nbadsecA+nbadsecD))
signs = 2*rgdn.integers(0,2,size=(nbadsecA+nbadsecD))-1 print(xposit+1)
dn = rgdn.integers(low=meanimg*0.4, high=meanimg*0.8, size=(nbadsecA+nbadsecD))*signs # signs = 2*rgdn.integers(0,2,size=(nbadsecA+nbadsecD))-1
if meanimg>0:
dn = rgdn.integers(low=meanimg*1.3+50, high=meanimg*2+150, size=(nbadsecA+nbadsecD)) #*signs
elif meanimg<0:
dn = rgdn.integers(low=meanimg*2-150, high=meanimg*1.3-50, size=(nbadsecA+nbadsecD)) #*signs
for badcoli in range(nbadsecA): for badcoli in range(nbadsecA):
GSImage.array[(ysize-collen[badcoli]):ysize,xposit[badcoli]:(xposit[badcoli]+1)] += np.random.random((collen[badcoli],1))*stdimg*3+dn[badcoli] GSImage.array[(ysize-collen[badcoli]):ysize,xposit[badcoli]:(xposit[badcoli]+1)] = (np.abs(np.random.normal(0, stdimg*2, (collen[badcoli],1)))+dn[badcoli])
for badcoli in range(nbadsecD): for badcoli in range(nbadsecD):
GSImage.array[0:collen[badcoli+nbadsecA],xposit[badcoli+nbadsecA]:(xposit[badcoli+nbadsecA]+1)] += np.random.random((collen[badcoli+nbadsecA],1))*stdimg*3+dn[badcoli+nbadsecA] GSImage.array[0:collen[badcoli+nbadsecA],xposit[badcoli+nbadsecA]:(xposit[badcoli+nbadsecA]+1)] = (np.abs(np.random.normal(0, stdimg*2, (collen[badcoli+nbadsecA],1)))+dn[badcoli+nbadsecA])
return GSImage return GSImage
...@@ -223,7 +229,7 @@ def PRNU_Img(xsize, ysize, sigma=0.01, seed=202101): ...@@ -223,7 +229,7 @@ def PRNU_Img(xsize, ysize, sigma=0.01, seed=202101):
return prnuimg return prnuimg
def NonLinearity(GSImage, beta1=1E-7, beta2=1E-10): def NonLinearity(GSImage, beta1=5E-7, beta2=0):
NonLinear_f = lambda x, beta_1, beta_2: x - beta_1*x*x + beta_2*x*x*x NonLinear_f = lambda x, beta_1, beta_2: x - beta_1*x*x + beta_2*x*x*x
GSImage.applyNonlinearity(NonLinear_f, beta1, beta2) GSImage.applyNonlinearity(NonLinear_f, beta1, beta2)
return GSImage return GSImage
......
...@@ -73,5 +73,5 @@ class Filter(object): ...@@ -73,5 +73,5 @@ class Filter(object):
def getPhotonE(self): def getPhotonE(self):
return photonEnergy(self.effective_wavelength) return photonEnergy(self.effective_wavelength)
def getSkyNoise(self, exptime, gain): def getSkyNoise(self, exptime, gain=1.):
return self.sky_background * exptime / gain return self.sky_background * exptime / gain
...@@ -97,6 +97,7 @@ class Observation(object): ...@@ -97,6 +97,7 @@ class Observation(object):
# Load SED # Load SED
if obj.type == 'star': if obj.type == 'star':
normF = chip.normF_star normF = chip.normF_star
#continue
try: try:
obj.load_SED( obj.load_SED(
survey_type=chip.survey_type, survey_type=chip.survey_type,
...@@ -108,6 +109,7 @@ class Observation(object): ...@@ -108,6 +109,7 @@ class Observation(object):
continue continue
elif obj.type == 'galaxy': # or obj.type == quasar elif obj.type == 'galaxy': # or obj.type == quasar
normF = chip.normF_galaxy normF = chip.normF_galaxy
# continue
obj.load_SED( obj.load_SED(
sed_path=sed_dir, sed_path=sed_dir,
survey_type=chip.survey_type, survey_type=chip.survey_type,
...@@ -116,6 +118,7 @@ class Observation(object): ...@@ -116,6 +118,7 @@ class Observation(object):
target_filt=filt) target_filt=filt)
elif obj.type == 'quasar': elif obj.type == 'quasar':
normF = chip.normF_galaxy normF = chip.normF_galaxy
# continue
obj.load_SED( obj.load_SED(
sed_path=sed_dir, sed_path=sed_dir,
survey_type=chip.survey_type, survey_type=chip.survey_type,
...@@ -200,7 +203,7 @@ class Observation(object): ...@@ -200,7 +203,7 @@ class Observation(object):
# Detector Effects # Detector Effects
# =========================================================== # ===========================================================
chip.img = chip.addNoise(config=self.config, tel=self.tel, filt=filt, img=chip.img, sky_map=sky_map) # chip.img = chip.addNoise(config=self.config, tel=self.tel, filt=filt, img=chip.img, sky_map=sky_map)
# chip.img = chip.addEffects(config=self.config, img=chip.img, chip_output=chip_output, filt=filt, pointing_ID=pointing_ID) # chip.img = chip.addEffects(config=self.config, img=chip.img, chip_output=chip_output, filt=filt, pointing_ID=pointing_ID)
# whether to output zero, dark, flat calibration images. # whether to output zero, dark, flat calibration images.
...@@ -214,7 +217,8 @@ class Observation(object): ...@@ -214,7 +217,8 @@ class Observation(object):
img_rot=img_rot, img_rot=img_rot,
pointing_ID=pointing_ID, pointing_ID=pointing_ID,
timestamp_obs=timestamp_obs, timestamp_obs=timestamp_obs,
pointing_type=pointing_type) pointing_type=pointing_type,
sky_map=sky_map)
if pointing_type == 'MS': if pointing_type == 'MS':
datetime_obs = datetime.fromtimestamp(timestamp_obs) datetime_obs = datetime.fromtimestamp(timestamp_obs)
...@@ -309,13 +313,20 @@ class Observation(object): ...@@ -309,13 +313,20 @@ class Observation(object):
sed_dir=self.path_dict["SED_dir"]) sed_dir=self.path_dict["SED_dir"])
print("finished running chip#%d..."%(chip.chipID), flush=True) print("finished running chip#%d..."%(chip.chipID), flush=True)
def runExposure_MPI_PointingList(self, ra_cen=None, dec_cen=None, pRange=None, timestamp_obs=[1621915200], pointing_type=['MS'], img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None): def runExposure_MPI_PointingList(self, ra_cen=None, dec_cen=None, pRange=None, timestamp_obs=np.array([1621915200]), pointing_type=np.array(['MS']), img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None):
comm = MPI.COMM_WORLD comm = MPI.COMM_WORLD
ind_thread = comm.Get_rank() ind_thread = comm.Get_rank()
num_thread = comm.Get_size() num_thread = comm.Get_size()
nchips_per_fp = len(self.chip_list) nchips_per_fp = len(self.chip_list)
# TEMP
if len(timestamp_obs) == 1:
timestamp_obs = np.tile(timestamp_obs, len(ra_cen))
pointing_type = np.tile(pointing_type, len(ra_cen))
timestamp_obs = timestamp_obs[pRange]
pointing_type = pointing_type[pRange]
ra_cen = ra_cen[pRange] ra_cen = ra_cen[pRange]
dec_cen = dec_cen[pRange] dec_cen = dec_cen[pRange]
...@@ -325,11 +336,6 @@ class Observation(object): ...@@ -325,11 +336,6 @@ class Observation(object):
else: else:
pStart = 0 pStart = 0
# TEMP
if len(timestamp_obs) == 1:
timestamp_obs *= len(pRange)
pointing_type *= len(pRange)
for ipoint in range(len(ra_cen)): for ipoint in range(len(ra_cen)):
for ichip in range(nchips_per_fp): for ichip in range(nchips_per_fp):
i = ipoint*nchips_per_fp + ichip i = ipoint*nchips_per_fp + ichip
......
...@@ -171,8 +171,8 @@ class PSFInterp(PSFModel): ...@@ -171,8 +171,8 @@ class PSFInterp(PSFModel):
img = galsim.ImageF(imPSF, scale=pixSize) img = galsim.ImageF(imPSF, scale=pixSize)
gsp = galsim.GSParams(folding_threshold=folding_threshold) gsp = galsim.GSParams(folding_threshold=folding_threshold)
self.psf = galsim.InterpolatedImage(img, gsparams=gsp) self.psf = galsim.InterpolatedImage(img, gsparams=gsp)
gaussPSF = galsim.Gaussian(sigma=0.04, gsparams=gsp) # gaussPSF = galsim.Gaussian(sigma=0.04, gsparams=gsp)
self.psf = galsim.Convolve(gaussPSF, self.psf) # self.psf = galsim.Convolve(gaussPSF, self.psf)
return self.PSFspin(x=px/0.01, y=py/0.01) return self.PSFspin(x=px/0.01, y=py/0.01)
return imPSF return imPSF
......
...@@ -9,6 +9,8 @@ data_dir = "/data/simudata/CSSOSDataProductsSims/data/" ...@@ -9,6 +9,8 @@ data_dir = "/data/simudata/CSSOSDataProductsSims/data/"
config_file = os.path.join(work_dir, "ObservationSim.cfg") config_file = os.path.join(work_dir, "ObservationSim.cfg")
config = ReadConfig(config_file) config = ReadConfig(config_file)
exp_time = float(config['exp_time'])
# Read Pointing list # Read Pointing list
pointing_file = os.path.join(data_dir, "pointing10_20210202.dat") pointing_file = os.path.join(data_dir, "pointing10_20210202.dat")
f = open(pointing_file, 'r') f = open(pointing_file, 'r')
...@@ -37,12 +39,16 @@ pointing_type = ['CAL']*ncal + pointing_type ...@@ -37,12 +39,16 @@ pointing_type = ['CAL']*ncal + pointing_type
t0 = datetime(2021, 5, 25, 12, 0, 0) t0 = datetime(2021, 5, 25, 12, 0, 0)
t = datetime.timestamp(t0) t = datetime.timestamp(t0)
timestamp_obs = [] timestamp_obs = []
delta_t = 10 # Time elapsed between exposures (minutes)
for i in range(len(pointing_type)): for i in range(len(pointing_type)):
timestamp_obs.append(t) timestamp_obs.append(t)
if pointing_type[i] == 'CAL': if pointing_type[i] == 'CAL':
t += 3 * 5 * 60 # 3 calibration exposure t += 3 * delta_t * 60 # 3 calibration exposure
elif pointing_type[i] == 'MS': elif pointing_type[i] == 'MS':
t += 5 * 60 t += delta_t * 60
timestamp_obs = np.array(timestamp_obs)
pointing_type = np.array(pointing_type)
# Define the range of pointing list # Define the range of pointing list
pRange = range(0, 2) pRange = range(0, 2)
\ No newline at end of file
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
##mpdboot -n 10 -f ./mpd.hosts ##mpdboot -n 10 -f ./mpd.hosts
##PBS -j oe ##PBS -j oe
#PBS -l nodes=comput110:ppn=80 #PBS -l nodes=comput108:ppn=80
#####PBS -q longq #####PBS -q longq
#PBS -q batch #PBS -q batch
#PBS -u fangyuedong #PBS -u fangyuedong
......
...@@ -20,4 +20,4 @@ gc.enable() ...@@ -20,4 +20,4 @@ gc.enable()
# Testing run pointing list (using MPI) # Testing run pointing list (using MPI)
obs = Observation(work_dir=work_dir, data_dir=data_dir) obs = Observation(work_dir=work_dir, data_dir=data_dir)
# obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type) # obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type)
obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=150.) obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=exp_time)
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
# Observation setting # Observation setting
date_obs 210525 # Observation date [yymmdd] date_obs 210525 # Observation date [yymmdd]
time_obs 120000 # Observation time [hhmmss] time_obs 120000 # Observation time [hhmmss]
exp_time 150. # Exposure time [seconds]
ra_center 60.0 # Telesscope pointing center [degree] (Default) ra_center 60.0 # Telesscope pointing center [degree] (Default)
dec_center -40.0 dec_center -40.0
psf_rcont 0.15,0.8 # Radius of the 80% flux concentration (for a Gaussian PSF) psf_rcont 0.15,0.8 # Radius of the 80% flux concentration (for a Gaussian PSF)
...@@ -18,7 +19,7 @@ survey_type Photometric # Survey simulation option: ...@@ -18,7 +19,7 @@ survey_type Photometric # Survey simulation option:
# 'Photometric': only run sims for photometric chips # 'Photometric': only run sims for photometric chips
# 'Spectroscopic': only run sims for spectroscopic chps # 'Spectroscopic': only run sims for spectroscopic chps
# 'All': run sims for all chips # 'All': run sims for all chips
psf_model Interp # which PSF model to use: psf_model Interp # Which PSF model to use:
# 'Gauss': simple gaussin profile # 'Gauss': simple gaussin profile
# 'Interp': Interpolated PSF from measured data # 'Interp': Interpolated PSF from measured data
...@@ -26,8 +27,9 @@ psf_model Interp # which PSF model to use: ...@@ -26,8 +27,9 @@ psf_model Interp # which PSF model to use:
# Instrument effects # Instrument effects
#=========================================================== #===========================================================
field_dist y # Y/N: Whether to add field distortion field_dist y # Y/N: Whether to add field distortion
abs_back y # Y: add sky background + sky noise & dark + dark noise add_back y # Y/N: Whether to add sky background
# N: only add sky noise and dark noise add_dark y # Y/N: Whether to add dark noise
add_readout y # Y/N: Whether to add read-out (Gaussian) noise
add_bias y # Y/N: Whether to add bias-level to image add_bias y # Y/N: Whether to add bias-level to image
shutter_effect y # Y/N: Whether to apply shutter effect shutter_effect y # Y/N: Whether to apply shutter effect
shutter_output n # Y/N: Whether to export shutter effect 16-bit image (<=65535) shutter_output n # Y/N: Whether to export shutter effect 16-bit image (<=65535)
...@@ -56,6 +58,7 @@ NDark 1 # Number of frames to be exported ...@@ -56,6 +58,7 @@ NDark 1 # Number of frames to be exported
NFlat 1 # Number of frames to be exported NFlat 1 # Number of frames to be exported
dark_exptime 300 # Exposure time for dark current frame dark_exptime 300 # Exposure time for dark current frame
flat_exptime 150 # Exposure time for flat field frame flat_exptime 150 # Exposure time for flat field frame
readout_time 40 # Time for read-out by each channel
#=========================================================== #===========================================================
# Shear method # Shear method
...@@ -85,6 +88,7 @@ seed_mock 12345678 # Seed for generating random values of ra, ...@@ -85,6 +88,7 @@ seed_mock 12345678 # Seed for generating random values of ra,
seed_star 121212121 # Seed for generating random redshift, sed_type for stars seed_star 121212121 # Seed for generating random redshift, sed_type for stars
seed_gal 212121212 # Seed for generating random redshift, sed_type for galaxies seed_gal 212121212 # Seed for generating random redshift, sed_type for galaxies
seed_Av 121212 # Seed for generating random intrinsic extinction seed_Av 121212 # Seed for generating random intrinsic extinction
seed_poisson 20210601 # Seed for Poisson noise
seed_CR 20210317 # Seed for generating random cosmic ray map seed_CR 20210317 # Seed for generating random cosmic ray map
seed_flat 20210101 # Seed for generating random flat field seed_flat 20210101 # Seed for generating random flat field
seed_prnu 20210102 # Seed for photo-response non-uniformity seed_prnu 20210102 # Seed for photo-response non-uniformity
...@@ -94,3 +98,4 @@ seed_biasNonUniform 20210203 # Seed for Bias nonuniformity ...@@ -94,3 +98,4 @@ seed_biasNonUniform 20210203 # Seed for Bias nonuniformity
seed_rnNonUniform 20210204 # Seed for ReadNoise nonuniformity seed_rnNonUniform 20210204 # Seed for ReadNoise nonuniformity
seed_badcolumns 20240309 # Initial Seed for Bad Columns seed_badcolumns 20240309 # Initial Seed for Bad Columns
seed_defective 20210304 # Seed for Defective(bad) pixels seed_defective 20210304 # Seed for Defective(bad) pixels
seed_readout 20210601 # Seed for Read-out gaussian noise
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment