Commit 33a3e5e3 authored by Fang Yuedong's avatar Fang Yuedong
Browse files

Merge remote-tracking branch 'origin/new_sim' into sim_scheduler

parents 3c45e974 2cd2bbb9
......@@ -60,6 +60,9 @@ class Catalog(CatalogBase):
with pkg_resources.path('Catalog.data', 'SLOAN_SDSS.g.fits') as filter_path:
self.normF_star = Table.read(str(filter_path))
with pkg_resources.path('Catalog.data', 'lsst_throuput_g.fits') as filter_path:
self.normF_galaxy = Table.read(str(filter_path))
self.config = config
self.chip = chip
......@@ -138,8 +141,8 @@ class Catalog(CatalogBase):
return None
###mock_stamp_START
elif obj.type == "stamp":
#return self.normF_galaxy ###normalize_filter for stamp
return None
return self.normF_galaxy ###normalize_filter for stamp
#return None
###mock_stamp_END
else:
return None
......@@ -197,8 +200,8 @@ class Catalog(CatalogBase):
for igals in range(ngals):
# # (TEST)
# if igals > 100:
# break
if igals > 2000:
break
param = self.initialize_param()
param['ra'] = ra_arr[igals]
......@@ -321,8 +324,8 @@ class Catalog(CatalogBase):
)
for istars in range(nstars):
# # (TEST)
# if istars > 100:
# break
if istars > 100:
break
param = self.initialize_param()
param['ra'] = ra_arr[istars]
......@@ -389,6 +392,9 @@ class Catalog(CatalogBase):
input_time_str=time_str
)
for iAGNs in range(nAGNs):
if iAGNs > 100:
break
param = self.initialize_param()
param['ra'] = ra_arr[iAGNs]
param['dec'] = dec_arr[iAGNs]
......@@ -425,13 +431,15 @@ class Catalog(CatalogBase):
self.ud = galsim.UniformDeviate(pix_id)
for istamp in range(nstamps):
print('DEBUG:::istamp=', istamp)
fitsfile = os.path.join(self.cat_dir, "stampCats/"+stamps['filename'][istamp].decode('utf-8'))
hdu=fitsio.open(fitsfile)
param = self.initialize_param()
param['id'] = hdu[0].header['index'] #istamp
param['star'] = 3 # Stamp type in .cat file
param['lensGalaxyID'] = hdu[0].header['lensGID']
###param['lensGalaxyID'] = hdu[0].header['lensGID']
param['ra'] = hdu[0].header['ra']
param['dec']= hdu[0].header['dec']
param['pixScale']= hdu[0].header['pixScale']
......@@ -440,9 +448,9 @@ class Catalog(CatalogBase):
#param['PA']= hdu[0].header['PA']
#param['bfrac']= hdu[0].header['bfrac']
#param['z']= hdu[0].header['z']
param['mag_use_normal'] = 22 #hdu[0].header['m_normal'] #gals['mag_true_g_lsst']
param['mag_use_normal'] = 20 #hdu[0].header['m_normal'] #gals['mag_true_g_lsst']
assert(stamps['lensGID'][istamp] == param['lensGalaxyID'])
###assert(stamps['lensGID'][istamp] == param['lensGalaxyID'])
# Apply astrometric modeling
# in C3 case only aberration
......
import os
import galsim
import numpy as np
import astropy.constants as cons
from astropy import wcs
from astropy.table import Table
import astropy.io.fits as fitsio
from ObservationSim.MockObject._util import magToFlux, VC_A, convolveGaussXorders, convolveImg
from ObservationSim.MockObject._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, \
......@@ -444,3 +446,88 @@ class MockObject(object):
sig_obj = np.std(stamp.array)
snr_obj = img_flux / sig_obj
return snr_obj
def drawObj_PSF(self, tel, pos_img, psf_model, bandpass_list, filt, chip, nphotons_tot=None, g1=0, g2=0,
exptime=150., fd_shear=None, chip_output=None):
if nphotons_tot == None:
nphotons_tot = self.getElectronFluxFilt(filt, tel, exptime)
# print("nphotons_tot = ", nphotons_tot)
try:
full = integrate_sed_bandpass(sed=self.sed, bandpass=filt.bandpass_full)
except Exception as e:
print(e)
if self.logger:
self.logger.error(e)
return 2, None
# Set Galsim Parameters
if self.getMagFilter(filt) <= 15:
folding_threshold = 5.e-4
else:
folding_threshold = 5.e-3
gsp = galsim.GSParams(folding_threshold=folding_threshold)
# Get real image position of object (deal with chip rotation w.r.t its center)
self.real_pos = self.getRealPos(chip.img, global_x=self.posImg.x, global_y=self.posImg.y,
img_real_wcs=self.chip_wcs)
x, y = self.real_pos.x + 0.5, self.real_pos.y + 0.5
x_nominal = int(np.floor(x + 0.5))
y_nominal = int(np.floor(y + 0.5))
dx = x - x_nominal
dy = y - y_nominal
offset = galsim.PositionD(dx, dy)
# Get real local wcs of object (deal with chip rotation w.r.t its center)
chip_wcs_local = self.chip_wcs.local(self.real_pos)
is_updated = 0
# Loop over all sub-bandpasses
for i in range(len(bandpass_list)):
bandpass = bandpass_list[i]
try:
sub = integrate_sed_bandpass(sed=self.sed, bandpass=bandpass)
except Exception as e:
print(e)
if self.logger:
self.logger.error(e)
continue
ratio = sub / full
if not (ratio == -1 or (ratio != ratio)):
nphotons = ratio * nphotons_tot
else:
continue
# Get PSF model
psf, pos_shear = psf_model.get_PSF(chip=chip, pos_img=pos_img, bandpass=bandpass,
folding_threshold=folding_threshold)
star_temp = psf.withFlux(nphotons)
if i==0:
star = star_temp
else:
star = star+star_temp
pixelScale = 0.074
stamp = star.drawImage(wcs=chip_wcs_local, offset=offset)
#stamp = star.drawImage(nx=256, ny=256, scale=pixelScale)
if np.sum(np.isnan(stamp.array)) > 0:
return None
fn = chip_output.subdir + "/psfIDW"
os.makedirs(fn, exist_ok=True)
fn = fn + "/ccd_{:}".format(chip.chipID)+"_psf_"+str(self.param['id'])+".fits"
if fn != None:
if os.path.exists(fn):
os.remove(fn)
hdu = fitsio.PrimaryHDU()
hdu.data = stamp.array
hdu.header.set('name', self.type)
hdu.header.set('pixScale', pixelScale)
hdu.header.set('objID', self.param['id'])
hdu.writeto(fn)
del stamp
return None
import os, sys
import random
import numpy as np
import galsim
import astropy.constants as cons
from astropy.table import Table
from scipy import interpolate
import astropy.io.fits as fitsio
import galsim
import gc
from ObservationSim.MockObject.MockObject import MockObject
from ObservationSim.MockObject._util import magToFlux,VC_A
from ObservationSim.MockObject.SpecDisperser import SpecDisperser
from ObservationSim.MockObject._util import eObs, integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG,convolveGaussXorders
class Stamp(MockObject):
def __init__(self, param):
super().__init__(param)
def __init__(self, param, logger=None):
super().__init__(param, logger=logger)
def unload_SED(self):
"""(Test) free up SED memory
......@@ -35,9 +30,9 @@ class Stamp(MockObject):
self.logger.error(e)
return False
nphotons_sum = 0
photons_list = []
xmax, ymax = 0, 0
#nphotons_sum = 0
#photons_list = []
#xmax, ymax = 0, 0
if self.getMagFilter(filt) <= 15:
folding_threshold = 5.e-4
......@@ -46,7 +41,7 @@ class Stamp(MockObject):
gsp = galsim.GSParams(folding_threshold=folding_threshold)
self.real_pos = self.getRealPos(chip.img, global_x=self.posImg.x, global_y=self.posImg.y,
img_real_wcs=self.real_wcs)
img_real_wcs=self.chip_wcs)
x, y = self.real_pos.x + 0.5, self.real_pos.y + 0.5
x_nominal = int(np.floor(x + 0.5))
......@@ -55,72 +50,237 @@ class Stamp(MockObject):
dy = y - y_nominal
offset = galsim.PositionD(dx, dy)
real_wcs_local = self.real_wcs.local(self.real_pos)
chip_wcs_local = self.chip_wcs.local(self.real_pos)
is_updated = 0
if fd_shear:
g1 += fd_shear.g1
g2 += fd_shear.g2
gal_shear = galsim.Shear(g1=g1, g2=g2)
for i in range(len(bandpass_list)):
bandpass = bandpass_list[i]
try:
sub = integrate_sed_bandpass(sed=self.sed, bandpass=bandpass)
except Exception as e:
print(e)
self.logger.error(e)
# return False
if self.logger:
self.logger.error(e)
continue
ratio = sub/full
if not (ratio == -1 or (ratio != ratio)):
nphotons = ratio * nphotons_tot
else:
# return False
continue
nphotons_sum += nphotons
#nphotons_sum += nphotons
psf, pos_shear = psf_model.get_PSF(chip=chip, pos_img=pos_img, bandpass=bandpass, folding_threshold=folding_threshold)
_gal = self.param['image']
galIm = galsim.ImageF(_gal, scale=self.param['pixScale'])
gal = galsim.InterpolatedImage(galIm)
gal = gal.withFlux(nphotons)
#gal_shear = galsim.Shear(g1=g1, g2=g2)
#gal = gal.shear(gal_shear)
galImg= galsim.ImageF(_gal, scale=self.param['pixScale'])
gal_temp= galsim.InterpolatedImage(galImg)
gal_temp= gal_temp.shear(gal_shear)
gal_temp= gal_temp.withFlux(nphotons)
gal = galsim.Convolve(psf, gal)
if fd_shear is not None:
gal = gal.shear(fd_shear)
gal_temp= galsim.Convolve(psf, gal_temp)
stamp = gal.drawImage(wcs=real_wcs_local, method='phot', offset=self.offset, save_photons=True)
xmax = max(xmax, stamp.xmax - stamp.xmin)
ymax = max(ymax, stamp.ymax - stamp.ymin)
photons = stamp.photons
photons.x += x_nominal
photons.y += y_nominal
photons_list.append(photons)
del gal
if i == 0:
gal = gal_temp
else:
gal = gal + gal_temp
# print('xmax = %d, ymax = %d '%(xmax, ymax))
stamp = gal.drawImage(wcs=chip_wcs_local, offset=offset)
if np.sum(np.isnan(stamp.array)) > 0:
# ERROR happens
return 2, pos_shear
stamp = galsim.ImageF(int(xmax*1.1), int(ymax*1.1))
stamp.wcs = real_wcs_local
stamp.setCenter(x_nominal, y_nominal)
bounds = stamp.bounds & galsim.BoundsI(0, chip.npix_x - 1, 0, chip.npix_y - 1)
if bounds.area() > 0:
chip.img.setOrigin(0, 0)
stamp[bounds] = chip.img[bounds]
for i in range(len(photons_list)):
if i == 0:
chip.sensor.accumulate(photons_list[i], stamp)
else:
chip.sensor.accumulate(photons_list[i], stamp, resume=True)
chip.img[bounds] += stamp[bounds]
is_updated = 1
chip.img.setOrigin(chip.bound.xmin, chip.bound.ymin)
del stamp
chip.img[bounds] = stamp[bounds]
if is_updated == 0:
print("fits obj %s missed"%(self.id))
if self.logger:
self.logger.info("fits obj %s missed"%(self.id))
return 0, pos_shear
chip.img.setOrigin(chip.bound.xmin, chip.bound.ymin)
return 1, pos_shear
def drawObj_slitless(self, tel, pos_img, psf_model, bandpass_list, filt, chip, nphotons_tot=None, g1=0, g2=0,
exptime=150., normFilter=None, grating_split_pos=3685, fd_shear=None):
if normFilter is not None:
norm_thr_rang_ids = normFilter['SENSITIVITY'] > 0.001
sedNormFactor = getNormFactorForSpecWithABMAG(ABMag=self.param['mag_use_normal'], spectrum=self.sed,
norm_thr=normFilter,
sWave=np.floor(normFilter[norm_thr_rang_ids][0][0]),
eWave=np.ceil(normFilter[norm_thr_rang_ids][-1][0]))
if sedNormFactor == 0:
return 2, None
else:
sedNormFactor = 1.
normalSED = Table(np.array([self.sed['WAVELENGTH'], self.sed['FLUX'] * sedNormFactor]).T,
names=('WAVELENGTH', 'FLUX'))
self.real_pos = self.getRealPos(chip.img, global_x=self.posImg.x, global_y=self.posImg.y,
img_real_wcs=self.chip_wcs)
x, y = self.real_pos.x + 0.5, self.real_pos.y + 0.5
x_nominal = int(np.floor(x + 0.5))
y_nominal = int(np.floor(y + 0.5))
dx = x - x_nominal
dy = y - y_nominal
offset = galsim.PositionD(dx, dy)
chip_wcs_local = self.chip_wcs.local(self.real_pos)
if self.getMagFilter(filt) <= 15:
folding_threshold = 5.e-4
else:
folding_threshold = 5.e-3
gsp = galsim.GSParams(folding_threshold=folding_threshold)
# nphotons_sum = 0
flat_cube = chip.flat_cube
xOrderSigPlus = {'A':1.3909419820029296,'B':1.4760376591236062,'C':4.035447379743442,'D':5.5684364343742825,'E':16.260021029735388}
grating_split_pos_chip = 0 + grating_split_pos
branges = np.zeros([len(bandpass_list), 2])
# print(hasattr(psf_model, 'bandranges'))
if hasattr(psf_model, 'bandranges'):
if psf_model.bandranges is None:
return 2, None
if len(psf_model.bandranges) != len(bandpass_list):
return 2, None
branges = psf_model.bandranges
else:
for i in range(len(bandpass_list)):
branges[i, 0] = bandpass_list[i].blue_limit * 10
branges[i, 1] = bandpass_list[i].red_limit * 10
for i in range(len(bandpass_list)):
# bandpass = bandpass_list[i]
brange = branges[i]
# psf, pos_shear = psf_model.get_PSF(chip=chip, pos_img=pos_img, bandpass=bandpass, folding_threshold=folding_threshold)
_gal = self.param['image']
galImg= galsim.ImageF(_gal, scale=self.param['pixScale'])
gal = galsim.InterpolatedImage(galImg)
# (TEST) Random knots
# knots = galsim.RandomKnots(npoints=100, profile=disk)
# kfrac = np.random.random()*(1.0 - self.bfrac)
# gal = self.bfrac * bulge + (1.0 - self.bfrac - kfrac) * disk + kfrac * knots
gal = gal.withFlux(tel.pupil_area * exptime)
if fd_shear:
g1 += fd_shear.g1
g2 += fd_shear.g2
gal_shear = galsim.Shear(g1=g1, g2=g2)
gal = gal.shear(gal_shear)
# gal = galsim.Convolve(psf, gal)
# if not big_galaxy: # Not apply PSF for very big galaxy
# gal = galsim.Convolve(psf, gal)
# # if fd_shear is not None:
# # gal = gal.shear(fd_shear)
starImg = gal.drawImage(wcs=chip_wcs_local, offset=offset,method = 'real_space')
origin_star = [y_nominal - (starImg.center.y - starImg.ymin),
x_nominal - (starImg.center.x - starImg.xmin)]
starImg.setOrigin(0, 0)
gal_origin = [origin_star[0], origin_star[1]]
gal_end = [origin_star[0] + starImg.array.shape[0] - 1, origin_star[1] + starImg.array.shape[1] - 1]
if gal_origin[1] < grating_split_pos_chip < gal_end[1]:
subSlitPos = int(grating_split_pos_chip - gal_origin[1] + 1)
## part img disperse
subImg_p1 = starImg.array[:, 0:subSlitPos]
star_p1 = galsim.Image(subImg_p1)
star_p1.setOrigin(0, 0)
origin_p1 = origin_star
xcenter_p1 = min(x_nominal,grating_split_pos_chip-1) - 0
ycenter_p1 = y_nominal-0
sdp_p1 = SpecDisperser(orig_img=star_p1, xcenter=xcenter_p1,
ycenter=ycenter_p1, origin=origin_p1,
tar_spec=normalSED,
band_start=brange[0], band_end=brange[1],
conf=chip.sls_conf[0],
isAlongY=0,
flat_cube=flat_cube)
# self.addSLStoChipImage(sdp=sdp_p1, chip=chip, xOrderSigPlus = xOrderSigPlus, local_wcs=chip_wcs_local)
pos_shear = self.addSLStoChipImageWithPSF(sdp=sdp_p1, chip=chip, pos_img_local=[xcenter_p1, ycenter_p1],
psf_model=psf_model, bandNo=i + 1,
grating_split_pos=grating_split_pos,
local_wcs=chip_wcs_local, pos_img = pos_img)
subImg_p2 = starImg.array[:, subSlitPos+1:starImg.array.shape[1]]
star_p2 = galsim.Image(subImg_p2)
star_p2.setOrigin(0, 0)
origin_p2 = [origin_star[0], grating_split_pos_chip]
xcenter_p2 = max(x_nominal, grating_split_pos_chip - 1) - 0
ycenter_p2 = y_nominal - 0
sdp_p2 = SpecDisperser(orig_img=star_p2, xcenter=xcenter_p2,
ycenter=ycenter_p2, origin=origin_p2,
tar_spec=normalSED,
band_start=brange[0], band_end=brange[1],
conf=chip.sls_conf[1],
isAlongY=0,
flat_cube=flat_cube)
# self.addSLStoChipImage(sdp=sdp_p2, chip=chip, xOrderSigPlus = xOrderSigPlus, local_wcs=chip_wcs_local)
pos_shear = self.addSLStoChipImageWithPSF(sdp=sdp_p2, chip=chip, pos_img_local=[xcenter_p2, ycenter_p2],
psf_model=psf_model, bandNo=i + 1,
grating_split_pos=grating_split_pos,
local_wcs=chip_wcs_local, pos_img = pos_img)
del sdp_p1
del sdp_p2
elif grating_split_pos_chip<=gal_origin[1]:
sdp = SpecDisperser(orig_img=starImg, xcenter=x_nominal - 0,
ycenter=y_nominal - 0, origin=origin_star,
tar_spec=normalSED,
band_start=brange[0], band_end=brange[1],
conf=chip.sls_conf[1],
isAlongY=0,
flat_cube=flat_cube)
# self.addSLStoChipImage(sdp=sdp, chip=chip, xOrderSigPlus = xOrderSigPlus, local_wcs=chip_wcs_local)
pos_shear = self.addSLStoChipImageWithPSF(sdp=sdp, chip=chip, pos_img_local=[x_nominal, y_nominal],
psf_model=psf_model, bandNo=i + 1,
grating_split_pos=grating_split_pos,
local_wcs=chip_wcs_local, pos_img = pos_img)
del sdp
elif grating_split_pos_chip>=gal_end[1]:
sdp = SpecDisperser(orig_img=starImg, xcenter=x_nominal - 0,
ycenter=y_nominal - 0, origin=origin_star,
tar_spec=normalSED,
band_start=brange[0], band_end=brange[1],
conf=chip.sls_conf[0],
isAlongY=0,
flat_cube=flat_cube)
# self.addSLStoChipImage(sdp=sdp, chip=chip, xOrderSigPlus = xOrderSigPlus, local_wcs=chip_wcs_local)
pos_shear = self.addSLStoChipImageWithPSF(sdp=sdp, chip=chip, pos_img_local=[x_nominal, y_nominal],
psf_model=psf_model, bandNo=i + 1,
grating_split_pos=grating_split_pos,
local_wcs=chip_wcs_local, pos_img = pos_img)
del sdp
del photons_list
del stamp
gc.collect()
return True, pos_shear
# print(self.y_nominal, starImg.center.y, starImg.ymin)
# del psf
return 1, pos_shear
......@@ -304,6 +304,20 @@ class Observation(object):
normFilter=norm_filt,
fd_shear=fd_shear)
if isUpdated == 1 and self.config["run_option"]["out_psf"]:
obj.drawObj_PSF(
tel=self.tel,
pos_img=pos_img,
psf_model=psf_model,
bandpass_list=filt.bandpass_sub_list,
filt=filt,
chip=chip,
g1=obj.g1,
g2=obj.g2,
exptime=pointing.exp_time,
fd_shear=fd_shear,
chip_output=chip_output)
if isUpdated == 1:
# TODO: add up stats
chip_output.cat_add_obj(obj, pos_img, pos_shear)
......
......@@ -9,9 +9,11 @@
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
work_dir: "/share/simudata/CSSOSDataProductsSims/data/CSSTSimImage_C6/"
work_dir: "/share/simudata/CSSOSDataProductsSims/data/CSSTSimImage_C8/"
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
run_name: "C6_fits_testRun"
run_name: "testRun_FGS"
project_cycle: 6
run_counter: 1
# Whether to use MPI
run_option:
......@@ -25,6 +27,9 @@ run_option:
# If yes, no imaging simulation will run
out_cat_only: NO
#output PSF
out_psf: YES
###############################################
# Catalog setting
###############################################
......@@ -46,13 +51,13 @@ catalog_options:
#stamp_SED:
# Only simulate stars?
star_only: NO
star_only: YES
# Only simulate galaxies?
galaxy_only: NO
# With stamp?
stamp_yes: YES
stamp_yes: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
......@@ -63,13 +68,23 @@ catalog_options:
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscopic": simulate slitless spectroscopic chips only
# "FGS": simulate FGS chips only (31-42)
# "All": simulate full focal plane
survey_type: "Photometric"
# "CALIBRATION": falt, bias, dark with or without postflash
survey_type: "All"
# "LED": ['LED1','LED2','LED3','LED4','LED5','LED6','LED7','LED8','LED9','LED10','LED11','LED12','LED13','LED14'] or null
# 'LED1': '275', 'LED2': '310', 'LED3': '430', 'LED4': '505', 'LED5': '545', 'LED6': '590', 'LED7': '670',
# 'LED8': '760', 'LED9': '940', 'LED10': '940', 'LED11': '1050', 'LED12': '1050','LED13': '340', 'LED14': '365'
# LED_TYPE: ['LED5']
# LED_TIME: [1.]
# # unit e- ,flat level
# FLAT_LEVEL: 20000
# FLAT_LEVEL_FIL: 'g'
# Exposure time [seconds]
exp_time: 150.
......@@ -79,19 +94,17 @@ obs_setting:
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 244.972773
dec_center: 39.895901
ra_center: 192.8595
dec_center: 27.1283
# Image rotation [degree]
image_rot: 109.59
image_rot: -113.4333
# (Optional) a file of point list
# (Optional) a file of point list
# if you just want to run default pointing:
# - pointing_dir: null
# - pointing_file: null
pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
pointing_file: "pointing_radec_246.5_40.dat"
# pointing_dir: null
# pointing_file: null
# Number of calibration pointings
np_cal: 0
......@@ -101,21 +114,25 @@ obs_setting:
# - run all pointings: null
# Note: only valid when a pointing list is specified
run_pointings: [0]
# Run specific chip(s):
# - give a list of indexes of chips: [ip_1, ip_2...]
# - run all chips: null
# Note: for all pointings
run_chips: [7]
run_chips: [39,40,41,42]
# Whether to enable astrometric modeling
enable_astrometric_model: False
enable_astrometric_model: True
# Whether to enable straylight model
enable_straylight_model: True
# Cut by saturation magnitude in which band?
cut_in_band: "z"
# saturation magnitude margin
mag_sat_margin: -2.5
# mag_sat_margin: -2.5
mag_sat_margin: -15.
# limiting magnitude margin
mag_lim_margin: +1.0
......@@ -138,7 +155,7 @@ psf_setting:
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"
psf_sls_dir: "/share/simudata/CSSOSDataProductsSims/data/SLS_PSF_PCA_fp/"
###############################################
# Shear setting
###############################################
......@@ -160,25 +177,27 @@ ins_effects:
# switches
# Note: bias_16channel, gain_16channel, and shutter_effect
# is currently not applicable to "FGS" observations
field_dist: OFF # Whether to add field distortions
add_back: OFF # Whether to add sky background
add_dark: OFF # Whether to add dark noise
add_readout: OFF # Whether to add read-out (Gaussian) noise
add_bias: OFF # Whether to add bias-level to images
bias_16channel: OFF # Whether to add different biases for 16 channels
gain_16channel: OFF # Whether to make different gains for 16 channels
shutter_effect: OFF # Whether to add shutter effect
flat_fielding: OFF # Whether to add flat-fielding effect
prnu_effect: OFF # Whether to add PRNU effect
non_linear: OFF # Whether to add non-linearity
cosmic_ray: OFF # Whether to add cosmic-ray
cray_differ: OFF # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: OFF # Whether to simulate CTE trails
saturbloom: OFF # Whether to simulate Saturation & Blooming
add_badcolumns: OFF # Whether to add bad columns
add_hotpixels: OFF # Whether to add hot pixels
add_deadpixels: OFF # Whether to add dead(dark) pixels
bright_fatter: OFF # Whether to simulate Brighter-Fatter (also diffusion) effect
field_dist: YES # Whether to add field distortions
add_back: YES # Whether to add sky background
add_dark: YES # Whether to add dark noise
add_readout: YES # Whether to add read-out (Gaussian) noise
add_bias: YES # Whether to add bias-level to images
bias_16channel: YES # Whether to add different biases for 16 channels
gain_16channel: YES # Whether to make different gains for 16 channels
shutter_effect: NO # Whether to add shutter effect
flat_fielding: YES # Whether to add flat-fielding effect
prnu_effect: YES # Whether to add PRNU effect
non_linear: YES # Whether to add non-linearity
cosmic_ray: YES # Whether to add cosmic-ray
cray_differ: YES # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: NO # Whether to simulate CTE trails, CTI_lgl_v0.3.tar.gz
saturbloom: YES # Whether to simulate Saturation & Blooming
add_badcolumns: YES # Whether to add bad columns
add_hotpixels: YES # Whether to add hot pixels
add_deadpixels: YES # Whether to add dead(dark) pixels
bright_fatter: YES # Whether to simulate Brighter-Fatter (also diffusion) effect
add_prescan: NO # Whether to add pre/over-scan
format_output: NO ##1*16 output
# Values:
# default values have been defined individually for each chip in:
......@@ -220,4 +239,4 @@ random_seeds:
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
...
unittest of PSF module:
--test_loadPSFSet.py
--testdata: S30x30_psfCube_1.h5 config_test.yaml
--test_PSFInterpModule_coverage.py
--testdata: S30x30_psfCube_1.h5 S20x20_psfCube_1.h5 config_test.yaml
--test_Convolve.py
--testdata: ccd1-w1
'''
test galsim.interpolatedImage & galsim.convolve
'''
import os
import unittest
import numpy as np
import matplotlib.pyplot as plt
import galsim
import scipy.io
from scipy import ndimage
def psfEncircle(img, fraction=0.8, psfSampleSizeInMicrons=2.5, focalLengthInMeters=28, cenPix=None):
#imgMaxPix_x, imgMaxPix_y = findMaxPix(img)
y,x = ndimage.center_of_mass(img) #y-rows, x-cols
imgMaxPix_x = x #int(x)
imgMaxPix_y = y #int(y)
if cenPix != None:
imgMaxPix_x = cenPix[0]
imgMaxPix_y = cenPix[1]
im1 = img.copy()
im1size = im1.shape
dis = np.zeros_like(img)
for irow in range(im1size[0]):
for icol in range(im1size[1]):
dx = icol - imgMaxPix_x
dy = irow - imgMaxPix_y
dis[irow, icol] = np.hypot(dx, dy)
nn = im1size[1]*im1size[0]
disX = dis.reshape(nn)
disXsortId = np.argsort(disX)
imgX = img.reshape(nn)
imgY = imgX[disXsortId]
psfFrac = np.cumsum(imgY)/np.sum(imgY)
ind = np.where(psfFrac > fraction)[0][0]
REE80 = np.rad2deg(dis[np.where(img == imgY[ind])]*psfSampleSizeInMicrons*1e-6/focalLengthInMeters)*3600
return REE80
def check_galsimConvolve(path=None, plotImage=True):
#load psf data
data=scipy.io.loadmat(path)
imPSF = data['psf']
pixSize = np.rad2deg(5.*1e-6/28)*3600
imPSF = imPSF/np.sum(imPSF)
#psf -> galsimInterpolatedImage
img = galsim.ImageF(imPSF, scale=pixSize)
imgt= galsim.InterpolatedImage(img)
#imPSFt = imgt.drawImage(nx=256, ny=256, scale=pixSize, method='no_pixel')
imPSFt = imgt.drawImage(nx=256, ny=256, scale=pixSize)
ree80 = psfEncircle(imPSF, fraction=0.8, psfSampleSizeInMicrons=5.)
ree80_pix = ree80/(np.rad2deg((5.*1e-6/28))*3600)
sliceX = slice(128-int(np.round(ree80_pix[0])), 128+int(np.round(ree80_pix[0]))+1, 1)
#set a point sorce
src = galsim.DeltaFunction(flux=1.0)
result = galsim.Convolve(src, imgt)
#drawImage with same pixSize
#tmp = result.drawImage(nx=256, ny=256, scale=pixSize, method='no_pixel')
tmp = result.drawImage(nx=256, ny=256, scale=pixSize)
if plotImage != True:
res = (imPSFt.array - imPSF)/imPSF
d0 = np.mean(res[sliceX, sliceX].flatten())
res = (tmp.array - imPSFt.array)/imPSFt.array
d1 = np.mean(res[sliceX, sliceX].flatten())
return d0, d1
#plot images
fig = plt.figure(figsize=(22, 5))
ax=plt.subplot(1,3,1)
plt.imshow(imPSF[128-10:128+10, 128-10:128+10])
plt.annotate("ORG", [0.1, 0.9],xycoords="axes fraction", fontsize=16, color="w")
plt.colorbar()
ax=plt.subplot(1,3,2)
plt.imshow(imPSFt.array[128-10:128+10, 128-10:128+10])
plt.annotate("InterpolatedImage", [0.1, 0.9],xycoords="axes fraction", fontsize=16, color="w")
plt.colorbar()
ax=plt.subplot(1,3,3)
plt.imshow(tmp.array[128-10:128+10, 128-10:128+10])
plt.annotate("ConvolvedImage", [0.1, 0.9],xycoords="axes fraction", fontsize=16, color="w")
plt.colorbar()
plt.savefig(OUTPUTPATH+'/fig_check_galsimConvolve_1.pdf')
fig = plt.figure(figsize=(13, 10))
ax=plt.subplot(2,2,1)
res = (imPSFt.array - imPSF)/imPSF
plt.imshow(res[128-10:128+10, 128-10:128+10])
plt.annotate("$\Delta_1$", [0.1, 0.9],xycoords="axes fraction", fontsize=16, color="w")
plt.colorbar()
ax=plt.subplot(2,2,2)
plt.hist(res[sliceX, sliceX].flatten(), alpha=0.75, bins=4)
#plt.annotate("$\Delta_1^{\\rm REE80}$", [0.1, 0.9],xycoords="axes fraction", fontsize=16, color="k")
plt.xlabel("$\Delta_1^{\\rm REE80}$", fontsize=16)
plt.ylabel("PDF", fontsize=16)
ax=plt.subplot(2,2,3)
res = (tmp.array - imPSFt.array)/imPSFt.array
plt.imshow(res[128-10:128+10, 128-10:128+10])
plt.annotate("$\Delta_2$", [0.1, 0.9],xycoords="axes fraction", fontsize=16, color="w")
plt.colorbar()
ax=plt.subplot(2,2,4)
plt.hist(res[sliceX, sliceX].flatten(), alpha=0.75, bins=4)
#plt.annotate("$\Delta_2^{\\rm REE80}$", [0.1, 0.9],xycoords="axes fraction", fontsize=16, color="k")
plt.xlabel("$\Delta_2^{\\rm REE80}$", fontsize=16)
plt.ylabel("PDF", fontsize=16)
plt.savefig(OUTPUTPATH+'/fig_check_galsimConvolve_2.pdf')
def check_galsimConvolveALL(dataPath):
d0 = np.zeros(900)
d1 = np.zeros(900)
for ipsf in range(1,901,1):
print("ipsf={:}".format(ipsf), end="\r")
psfPath = dataPath+"/ccd1-w1/psf_{:}_centroidWgt_BC.mat".format(ipsf)
t0,t1=check_galsimConvolve(path = psfPath, plotImage=False)
d0[ipsf-1] = t0
d1[ipsf-1] = t1
fig = plt.figure(figsize=(12,6))
ax = plt.subplot(1,2,1)
#plt.scatter(np.linspace(1,900,900), d0)
plt.hist(d0, bins=8, alpha=0.75)
plt.xlabel("mean($\Delta_1^{\\rm REE80}$)", fontsize=16)
plt.ylabel("PDF", fontsize=16)
ax = plt.subplot(1,2,2)
#plt.scatter(np.linspace(1,900,900), d1)
plt.hist(d1, bins=8, alpha=0.75)
plt.xlabel("mean($\Delta_2^{\\rm REE80}$)", fontsize=16)
plt.ylabel("PDF", fontsize=16)
plt.savefig(OUTPUTPATH+'/fig_check_galsimConvolveALL.pdf')
class testConvolve(unittest.TestCase):
def __init__(self, methodName='runTest'):
super(testConvolve,self).__init__(methodName)
self.dataPath = os.path.join(os.getenv('UNIT_TEST_DATA_ROOT'), 'csst_fz_gc1')
global OUTPUTPATH
OUTPUTPATH = os.path.join(self.dataPath, 'outputs')
def test_galsimConvolve(self):
ipsf = 1
psfPath = self.dataPath+"/ccd1-w1/psf_{:}_centroidWgt_BC.mat".format(ipsf)
check_galsimConvolve(path = psfPath)
def test_galsimConvolveALL(self):
check_galsimConvolveALL(dataPath=self.dataPath)
if __name__ == "__main__":
unittest.main()
......@@ -35,7 +35,7 @@ def psfSecondMoments(psfMat, cenX, cenY, pixSize=1):
pxs = 2.5 #microns
apr = np.deg2rad(apr/3600.)*fl*1e6
apr = apr/pxs
apr = np.int(np.ceil(apr))
apr = int(np.ceil(apr))
I = psfMat
ncol = I.shape[1]
......@@ -90,7 +90,7 @@ def test_psfEll(iccd, iwave, psfMat):
#print('ell======', ipsf, np.sqrt(e1**2 + e2**2))
#######
arr = [imx, imy, psf_e1, psf_e2, psf_sz]
np.save('data/psfEll{:}_{:}_{:}'.format(int(np.sqrt(npsf)),iccd, iwave), arr)
np.save(OUTPUTPATH+'/psfEll{:}_{:}_{:}'.format(int(np.sqrt(npsf)),iccd, iwave), arr)
def test_psfEllPlot(OVERPLOT=False):
......@@ -99,7 +99,7 @@ def test_psfEllPlot(OVERPLOT=False):
prefix = 'psfEll30'
iccd = 1
iwave= 1
data = np.load('data/'+prefix+'_1_1.npy')
data = np.load(OUTPUTPATH+'/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
......@@ -136,7 +136,7 @@ def test_psfEllPlot(OVERPLOT=False):
if OVERPLOT == True:
prefix = 'psfEll20'
data = np.load('data/'+prefix+'_1_1.npy')
data = np.load(OUTPUTPATH+'/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
......@@ -156,7 +156,7 @@ def test_psfEllPlot(OVERPLOT=False):
plt.gca().set_aspect(1)
if OVERPLOT == True:
prefix = 'psfEllOP'
plt.savefig('figs/'+prefix+'_iccd{:}.pdf'.format(iccd))
plt.savefig(OUTPUTPATH+'/'+prefix+'_iccd{:}.pdf'.format(iccd))
def test_psfIDW(iccd, iwave, psfMatA, chip, psfMatB):
......@@ -176,7 +176,7 @@ def test_psfIDW(iccd, iwave, psfMatA, chip, psfMatB):
print('ipsf:', ipsf, end='\r', flush=True)
tpos_img = pos_img(psfMatA.cen_col[iwave-1, ipsf-1], psfMatA.cen_row[iwave-1, ipsf-1])
psfIDW = psfMatB.get_PSF(chip, tpos_img, bandpass, galsimGSObject=False, findNeighMode='treeFind')
np.save('figs/psfIDW/psfIDW_{:}_{:}_{:}'.format(iccd, iwave, ipsf), psfIDW)
np.save(OUTPUTPATH+'/psfIDW_{:}_{:}_{:}'.format(iccd, iwave, ipsf), psfIDW)
cenX = 256
cenY = 256
......@@ -185,14 +185,14 @@ def test_psfIDW(iccd, iwave, psfMatA, chip, psfMatB):
psf_e2[ipsf-1] = e2
psf_sz[ipsf-1] = sz
arr = [psf_e1, psf_e2, psf_sz]
np.save('data/psfEll20IDW_{:}_{:}'.format(iccd, iwave), arr)
np.save(OUTPUTPATH+'/psfEll20IDW_{:}_{:}'.format(iccd, iwave), arr)
def test_psfResidualPlot(iccd, iwave, ipsf, psfMatA):
psfMat_iwave = psfMatA.psfMat[iwave-1, :,:,:]
psfMatORG = psfMat_iwave[ipsf-1, :, :]
psfMatIDW = np.load('figs/psfIDW/psfIDW_{:}_{:}_{:}.npy'.format(iccd, iwave, ipsf))
psfMatIDW = np.load(OUTPUTPATH+'/psfIDW_{:}_{:}_{:}.npy'.format(iccd, iwave, ipsf))
npix = psfMatORG.shape[0]
pixCutEdge= int(npix/2-15)
......@@ -238,7 +238,7 @@ def test_psfResidualPlot(iccd, iwave, ipsf, psfMatA):
cbar.ax.set_yticklabels(['$10^{-5}$','$10^{-4}$','$10^{-3}$','$10^{-2}$', '$10^{-1}$'])
print(np.max((psfMatORG-psfMatIDW)))
plt.savefig('figs/psfResidual_iccd{:}.pdf'.format(iccd))
plt.savefig(OUTPUTPATH+'/psfResidual_iccd{:}.pdf'.format(iccd))
def test_psfEllIDWPlot(OVERPLOT=False):
......@@ -247,7 +247,7 @@ def test_psfEllIDWPlot(OVERPLOT=False):
prefix = 'psfEll20'
iccd = 1
iwave= 1
data = np.load('data/'+prefix+'_1_1.npy')
data = np.load(OUTPUTPATH+'/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
......@@ -284,7 +284,7 @@ def test_psfEllIDWPlot(OVERPLOT=False):
if OVERPLOT == True:
prefix = 'psfEll20IDW'
data = np.load('data/'+prefix+'_1_1.npy')
data = np.load(OUTPUTPATH+'/'+prefix+'_1_1.npy')
#imx= data[0]
#imy= data[1]
psf_e1 = data[0]
......@@ -304,7 +304,7 @@ def test_psfEllIDWPlot(OVERPLOT=False):
plt.gca().set_aspect(1)
if OVERPLOT == True:
prefix = 'psfEllOPIDW'
plt.savefig('figs/'+prefix+'_iccd{:}.pdf'.format(iccd))
plt.savefig(OUTPUTPATH+'/'+prefix+'_iccd{:}.pdf'.format(iccd))
def test_psfdEllabsPlot(iccd):
......@@ -313,7 +313,7 @@ def test_psfdEllabsPlot(iccd):
prefix = 'psfEll20'
#iccd = 1
#iwave= 1
data = np.load('data/'+prefix+'_{:}_1.npy'.format(iccd))
data = np.load(OUTPUTPATH+'/'+prefix+'_{:}_1.npy'.format(iccd))
imx= data[0]
imy= data[1]
psf_e1 = data[2]
......@@ -329,7 +329,7 @@ def test_psfdEllabsPlot(iccd):
##############################
prefix = 'psfEll20IDW'
data = np.load('data/'+prefix+'_{:}_1.npy'.format(iccd))
data = np.load(OUTPUTPATH+'/'+prefix+'_{:}_1.npy'.format(iccd))
#imx= data[0]
#imy= data[1]
psf_e1 = data[0]
......@@ -360,28 +360,33 @@ def test_psfdEllabsPlot(iccd):
#plt.ylim([-0.0018, 0.0018])
plt.xlabel('$\epsilon_{\\rm ORG}$')
plt.ylabel('$\Delta$')
plt.savefig('figs/psfEllOPIDWPDF_{:}.pdf'.format(iccd))
plt.savefig(OUTPUTPATH+'/psfEllOPIDWPDF_{:}.pdf'.format(iccd))
fig=plt.figure(figsize=(6, 6))
plt.hist((szY-szX)/szX, bins=20, color='r', alpha=0.5)
plt.xlabel('$(R_{\\rm IDW}-R_{\\rm ORG})/R_{\\rm ORG}$')
plt.ylabel('PDF')
plt.savefig('figs/psfEllOPIDWPDF_dsz_{:}.pdf'.format(iccd))
plt.savefig(OUTPUTPATH+'/psfEllOPIDWPDF_dsz_{:}.pdf'.format(iccd))
class PSFInterpModule_coverage(unittest.TestCase):
def __init__(self, methodName='runTest'):
super(PSFInterpModule_coverage,self).__init__(methodName)
self.dataPath = os.path.join(os.getenv('UNIT_TEST_DATA_ROOT'), 'csst_fz_gc1')
global OUTPUTPATH
OUTPUTPATH = os.path.join(self.dataPath, 'outputs')
def test_psfEll_(self):
iccd = 1
iwave= 1
config_file = "/public/home/weichengliang/CSST_git/newVersion/CSST/config/config_C3.yaml"
config_file = os.path.join(self.dataPath, 'config_test.yaml')
chip = defineCCD(iccd, config_file)
print(chip.chipID)
print(chip.cen_pix_x, chip.cen_pix_y)
ipath = '/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCubeTest'
psfMatA = PSFInterp(chip, npsf=400, PSF_data_file=ipath, PSF_data_prefix="S20x20_")
psfMatB = PSFInterp(chip, npsf=900, PSF_data_file=ipath, PSF_data_prefix="S30x30_")
psfMatA = PSFInterp(chip, npsf=400, PSF_data_file=self.dataPath, PSF_data_prefix="S20x20_")
psfMatB = PSFInterp(chip, npsf=900, PSF_data_file=self.dataPath, PSF_data_prefix="S30x30_")
test_psfEll(iccd, iwave, psfMatA)
test_psfEll(iccd, iwave, psfMatB)
......
......@@ -26,15 +26,14 @@ def defineCCD(iccd, config_file):
#chip = Chip(chipID=iccd, ccdEffCurve_dir=path_dict["ccd_dir"], CRdata_dir=path_dict["CRdata_dir"], normalize_dir=path_dict["normalize_dir"], sls_dir=path_dict['sls_dir'], config=config)
return chip
def loadPSFSet(iccd):
config_file = "/public/home/weichengliang/CSST_git/newVersion/CSST/config/config_C3.yaml"
def loadPSFSet(iccd, dataPath):
config_file = os.path.join(dataPath, 'config_test.yaml')
chip = defineCCD(iccd, config_file)
print(chip.chipID)
print(chip.cen_pix_x, chip.cen_pix_y)
ipath = '/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube'
psfMat= PSFInterp(chip, npsf=900, PSF_data_file=ipath, PSF_data_prefix="")
psfSet= psfMat._loadPSF(iccd, ipath, PSF_data_prefix="")
psfMat= PSFInterp(chip, npsf=900, PSF_data_file=dataPath, PSF_data_prefix="S30x30_")
psfSet= psfMat._loadPSF(iccd, dataPath, PSF_data_prefix="S30x30_")
twave = 0 #[0...3]
tpsf = 0 #[0...899]
......@@ -50,9 +49,13 @@ def loadPSFSet(iccd):
class PSFInterpModule_coverage(unittest.TestCase):
def __init__(self, methodName='runTest'):
super(PSFInterpModule_coverage,self).__init__(methodName)
self.dataPath = os.path.join(os.getenv('UNIT_TEST_DATA_ROOT'), 'csst_fz_gc1')
def test_psfEll_(self):
iccd = 1 #[1...30]
psfSet = loadPSFSet(iccd)
psfSet = loadPSFSet(iccd, dataPath=self.dataPath)
if __name__ == '__main__':
......
import unittest
import sys,os,math
from itertools import islice
#import mpi4py.MPI as MPI
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
mpl.use('Agg')
import scipy.io
from scipy import ndimage
#sys.path.append("/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_v4_20210326")
sys.path.append("../")
from ObservationSim.PSF.PSFInterp import PSFConfig as myConfig
#import PSFUtil as myUtil
NPSF = 400
##############################
###计算PSF椭率###
def psfSecondMoments(psfMat, cenX, cenY, pixSize=1):
apr = 0.5 #arcsec, 0.5角秒内测量
fl = 28. #meters
pxs = 2.5 #microns
apr = np.deg2rad(apr/3600.)*fl*1e6
apr = apr/pxs
apr = np.int(np.ceil(apr))
I = psfMat
ncol = I.shape[1]
nrow = I.shape[0]
w = 0.0
w11 = 0.0
w12 = 0.0
w22 = 0.0
for icol in range(ncol):
for jrow in range(nrow):
x = icol*pixSize - cenX
y = jrow*pixSize - cenY
rr = np.sqrt(x*x + y*y)
wgt= 0.0
if rr <= apr:
wgt = 1.0
w += I[jrow, icol]*wgt
w11 += x*x*I[jrow, icol]*wgt
w12 += x*y*I[jrow, icol]*wgt
w22 += y*y*I[jrow, icol]*wgt
w11 /= w
w12 /= w
w22 /= w
sz = w11 + w22
e1 = (w11 - w22)/sz
e2 = 2.0*w12/sz
return sz, e1, e2
##############################
##############################
'''
def assignTasks(npsf, NTasks, ThisTask):
npsfPerTasks = int(npsf/NTasks)
iStart= 0 + npsfPerTasks*ThisTask
iEnd = npsfPerTasks + npsfPerTasks*ThisTask
if ThisTask == NTasks:
iEnd = npsf
return iStart, iEnd
'''
#def test_psfEll(iccd, iwave, psfPath, ThisTask, NTasks):
def test_psfEll(iccd, iwave, psfPath):
nccd = 30
npsf = NPSF
#iStart, iEnd = assignTasks(npsf, NTasks, ThisTask)
imx = np.zeros(npsf)
imy = np.zeros(npsf)
psf_e1 = np.zeros(npsf)
psf_e2 = np.zeros(npsf)
psf_sz = np.zeros(npsf)
#for ipsf in range(iStart+1, iEnd+1):
for ipsf in range(1, 401):
if ipsf != 1:
continue
print('ipsf-{:}'.format(ipsf), end='\r')
psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPath, InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
imx[ipsf-1] = psfInfo['image_x']+psfInfo['centroid_x']
imy[ipsf-1] = psfInfo['image_y']+psfInfo['centroid_y']
psfMat = psfInfo['psfMat']
cenX = 256
cenY = 256
sz, e1, e2 = psfSecondMoments(psfMat, cenX, cenY, pixSize=1)
psf_e1[ipsf-1] = e1
psf_e2[ipsf-1] = e2
psf_sz[ipsf-1] = sz
print('test:' ,sz, e1, e2)
#######
#comm.barrier()
#imx = comm.allreduce(imx, op=MPI.SUM)
#imy = comm.allreduce(imy, op=MPI.SUM)
#psf_e1 = comm.allreduce(psf_e1, op=MPI.SUM)
#psf_e2 = comm.allreduce(psf_e2, op=MPI.SUM)
#psf_sz = comm.allreduce(psf_sz, op=MPI.SUM)
#comm.barrier()
#if ThisTask == 0:
# arr = [imx, imy, psf_e1, psf_e2, psf_sz]
# np.save('/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_v4_20210326/test4report/data/psfEll20_{:}_{:}'.format(iccd, iwave), arr)
def test_psfEllPlot(OVERPLOT=False):
#if ThisTask == 0:
if True:
prefix = 'psfEll30'
iccd = 1
iwave= 1
data = np.load('data/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
psf_e2 = data[3]
print(np.shape(imx))
npsf = np.shape(imx)[0]
plt.cla()
plt.close("all")
fig = plt.figure(figsize=(12,12))
plt.plot(imx, imy, 'r.')
plt.savefig('figs/psfPos.pdf')
#######
fig = plt.figure(figsize=(12, 12))
plt.subplots_adjust(wspace=0.1, hspace=0.1)
ax = plt.subplot(1, 1, 1)
for ipsf in range(npsf):
plt.plot(imx[ipsf], imy[ipsf], 'r.')
ang = np.arctan2(psf_e2[ipsf], psf_e1[ipsf])/2
ell = np.sqrt(psf_e1[ipsf]**2 + psf_e2[ipsf]**2)
ell *= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
plt.plot([imx[ipsf]-lcos, imx[ipsf]+lcos],[imy[ipsf]-lsin, imy[ipsf]+lsin],'r', lw=2)
###########
ang = 0.
ell = 0.05
ell*= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
plt.plot([imx[898]-lcos, imx[898]+lcos],[imy[898]+5.-lsin, imy[898]+5.+lsin],'k', lw=2)
plt.annotate('{:}'.format(ell/15), (imx[898]-2., imy[898]+6.), xycoords='data', fontsize=10)
plt.xlabel('CCD X (mm)')
plt.ylabel('CCD Y (mm)')
if OVERPLOT == True:
prefix = 'psfEll20'
data = np.load('data/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
psf_e2 = data[3]
npsf = np.shape(imx)[0]
for ipsf in range(npsf):
plt.plot(imx[ipsf], imy[ipsf], 'b.')
ang = np.arctan2(psf_e2[ipsf], psf_e1[ipsf])/2
ell = np.sqrt(psf_e1[ipsf]**2 + psf_e2[ipsf]**2)
ell *= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
plt.plot([imx[ipsf]-lcos, imx[ipsf]+lcos],[imy[ipsf]-lsin, imy[ipsf]+lsin],'b', lw=2)
plt.gca().set_aspect(1)
if OVERPLOT == True:
prefix = 'psfEllOP'
plt.savefig('figs/'+prefix+'_iccd{:}.pdf'.format(iccd))
class PSFMatsEll_coverage(unittest.TestCase):
def test_psfEll_(self):
#comm = MPI.COMM_WORLD
#ThisTask = comm.Get_rank()
#NTasks = comm.Get_size()
print('#####haha#####')
iccd = 1
iwave= 1
psfPath = '/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/CSSOS_psf_20210326/CSST_psf_ciomp_20x20field'
#test_psfEll(iccd, iwave, psfPath, ThisTask, NTasks)
test_psfEll(iccd, iwave, psfPath)
test_psfEllPlot(OVERPLOT=True)
##############################
##############################
##############################
if __name__=='__main__':
unittest.main()
import unittest
import sys,os,math
from itertools import islice
import mpi4py.MPI as MPI
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
mpl.use('Agg')
import scipy.io
from scipy import ndimage
#sys.path.append("/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_v4_20210326")
# sys.path.append("../")
from ObservationSim.PSF.PSFInterp import PSFConfig as myConfig
from ObservationSim.PSF.PSFInterp import PSFUtil as myUtil
NPSF = 400
##############################
###计算PSF椭率###
def psfSecondMoments(psfMat, cenX, cenY, pixSize=1):
apr = 0.5 #arcsec, 0.5角秒内测量
fl = 28. #meters
pxs = 2.5 #microns
apr = np.deg2rad(apr/3600.)*fl*1e6
apr = apr/pxs
apr = np.int(np.ceil(apr))
I = psfMat
ncol = I.shape[1]
nrow = I.shape[0]
w = 0.0
w11 = 0.0
w12 = 0.0
w22 = 0.0
for icol in range(ncol):
for jrow in range(nrow):
x = icol*pixSize - cenX
y = jrow*pixSize - cenY
rr = np.sqrt(x*x + y*y)
wgt= 0.0
if rr <= apr:
wgt = 1.0
w += I[jrow, icol]*wgt
w11 += x*x*I[jrow, icol]*wgt
w12 += x*y*I[jrow, icol]*wgt
w22 += y*y*I[jrow, icol]*wgt
w11 /= w
w12 /= w
w22 /= w
sz = w11 + w22
e1 = (w11 - w22)/sz
e2 = 2.0*w12/sz
return sz, e1, e2
##############################
##############################
'''
def assignTasks(npsf, NTasks, ThisTask):
npsfPerTasks = int(npsf/NTasks)
iStart= 0 + npsfPerTasks*ThisTask
iEnd = npsfPerTasks + npsfPerTasks*ThisTask
if ThisTask == NTasks:
iEnd = npsf
return iStart, iEnd
'''
#def test_psfIDW(iccd, iwave, psfPath, ThisTask, NTasks):
def test_psfIDW(iccd, iwave, psfPath):
nccd = 30
npsfA = 900
npsfB = 400
#iStart, iEnd = assignTasks(400, NTasks, ThisTask)
psfPathA = psfPath+'_30x30field'
psfPathB = psfPath+'_20x20field'
imxA = np.zeros(npsfA)
imyA = np.zeros(npsfA)
psfA = np.zeros([npsfA, 512, 512])
imxB = np.zeros(npsfB)
imyB = np.zeros(npsfB)
psfB = np.zeros([npsfB, 512, 512])
#for ipsf in range(iStart+1, iEnd+1):
for ipsf in range(1, npsfA+1):
print('ipsfA:', ipsf, end='\r', flush=True)
psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPathA, InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
imxA[ipsf-1] = psfInfo['image_x'] + psfInfo['centroid_x']
imyA[ipsf-1] = psfInfo['image_y'] + psfInfo['centroid_y']
psfA[ipsf-1, :, :] = psfInfo['psfMat']
for ipsf in range(1, npsfB+1):
print('ipsfB:', ipsf, end='\r', flush=True)
psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPathB, InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
imxB[ipsf-1] = psfInfo['image_x'] + psfInfo['centroid_x']
imyB[ipsf-1] = psfInfo['image_y'] + psfInfo['centroid_y']
psfB[ipsf-1, :, :] = psfInfo['psfMat']
#myConfig.psfMaker_IDW(px, py, PSFMat, cen_col, cen_row, IDWindex=2, OnlyNeighbors=True, hoc=None, hoclist=None, PSFCentroidWgt=False)
for ipsf in range(npsfB):
print('ipsf:', ipsf, end='\r', flush=True)
px = imxB[ipsf]
py = imyB[ipsf]
cen_col = imxA
cen_row = imyA
PSFMat = psfA
psfIDW = myConfig.psfMaker_IDW(px, py, PSFMat, cen_col, cen_row, IDWindex=2, OnlyNeighbors=True, hoc=None, hoclist=None, PSFCentroidWgt=False)
np.save('figs/psfIDW/psfIDW_{:}_{:}_{:}'.format(iccd, iwave, ipsf+1), psfIDW)
#def test_psfEll(iccd, iwave, ThisTask, NTasks):
def test_psfEll(iccd, iwave):
nccd = 30
npsf = 400
#iStart, iEnd = assignTasks(npsf, NTasks, ThisTask)
#imx = np.zeros(npsf)
#imy = np.zeros(npsf)
psf_e1 = np.zeros(npsf)
psf_e2 = np.zeros(npsf)
psf_sz = np.zeros(npsf)
#for ipsf in range(iStart+1, iEnd+1):
for ipsf in range(1, 401):
if ipsf > 1:
continue
print('ipsf-{:}'.format(ipsf), end='\r')
#psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPath, InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
#psfMat = psfInfo['psfMat']
#imx[ipsf-1] = psfInfo['image_x']+psfInfo['centroid_x']
#imy[ipsf-1] = psfInfo['image_y']+psfInfo['centroid_y']
psfMat = np.load('figs/psfIDW/psfIDW_{:}_{:}_{:}.npy'.format(iccd, iwave, ipsf)) #psfInfo['psfMat']
cenX = 256
cenY = 256
sz, e1, e2 = psfSecondMoments(psfMat, cenX, cenY, pixSize=1)
psf_e1[ipsf-1] = e1
psf_e2[ipsf-1] = e2
psf_sz[ipsf-1] = sz
#######
#comm.barrier()
#imx = comm.allreduce(imx, op=MPI.SUM)
#imy = comm.allreduce(imy, op=MPI.SUM)
#psf_e1 = comm.allreduce(psf_e1, op=MPI.SUM)
#psf_e2 = comm.allreduce(psf_e2, op=MPI.SUM)
#psf_sz = comm.allreduce(psf_sz, op=MPI.SUM)
#comm.barrier()
#if ThisTask == 0:
# arr = [psf_e1, psf_e2, psf_sz]
# np.save('/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_v4_20210326/test4report/data/psfEll20IDW_{:}_{:}'.format(iccd, iwave), arr)
'''
def test_psfResidualCalc(iccd, iwave, ipsf, psfPath):
psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPath+'_20x20field', InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
psfMatORG = psfInfo['psfMat']
psfMatIDW = np.load('/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_v4_20210326/test4report/figs/psfIDW/psfIDW_{:}_{:}_{:}.npy'.format(iccd, iwave, ipsf))
_,tREE80 = myUtil.psfEncircle(psfMatORG, fraction=0.8, psfSampleSizeInMicrons=2.5, focalLengthInMeters=28, cenPix=None)
tREE80_pix = np.int32(np.ceil(tREE80/(0.074/2/2))[0])
#print(tREE80, np.ceil(tREE80/(0.074/2/2)), tREE80_pix)
timg0 = psfMatORG[256-tREE80_pix:256+tREE80_pix, 256-tREE80_pix:256+tREE80_pix]
timg1 = psfMatIDW[256-tREE80_pix:256+tREE80_pix, 256-tREE80_pix:256+tREE80_pix]
#print("residual::", np.max((timg1-timg0)/timg0), np.min((timg1-timg0)/timg0), np.mean((timg1-timg0)/timg0))
return np.mean((timg1-timg0)/timg0)
'''
def test_psfResidualPlot(iccd, iwave, ipsf, psfPath):
psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPath+'_20x20field', InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
psfMatORG = psfInfo['psfMat']
psfMatIDW = np.load('figs/psfIDW/psfIDW_{:}_{:}_{:}.npy'.format(iccd, iwave, ipsf))
npix = psfMatORG.shape[0]
pixCutEdge= int(npix/2-15)
img0 = psfMatORG[pixCutEdge:npix-pixCutEdge, pixCutEdge:npix-pixCutEdge]
img1 = psfMatIDW[pixCutEdge:npix-pixCutEdge, pixCutEdge:npix-pixCutEdge]
imgX = (img1 - img0)/img0
img0 = np.log10(img0)
img1 = np.log10(img1)
imgX = np.log10(np.abs(imgX))
fig = plt.figure(figsize=(18,4))
ax = plt.subplot(1,3,1)
plt.imshow(img0, origin='lower', vmin=-7, vmax=-1.3)
plt.plot([npix/2-pixCutEdge, npix/2-pixCutEdge],[0, (npix/2-pixCutEdge)*2-1],'w--')
plt.plot([0, (npix/2-pixCutEdge)*2-1],[npix/2-pixCutEdge, npix/2-pixCutEdge],'w--')
plt.annotate('ORG', [0,(npix/2-pixCutEdge)*2-5], c='w', size=15)
cticks=[-7, -6, -5, -4, -3, -2, -1]
cbar = plt.colorbar(ticks=cticks)
cbar.ax.set_yticklabels(['$10^{-7}$', '$10^{-6}$', '$10^{-5}$','$10^{-4}$','$10^{-3}$','$10^{-2}$', '$10^{-1}$'])
print(img0.min(), img0.max())
ax = plt.subplot(1,3,2)
plt.imshow(img1, origin='lower', vmin=-7, vmax=-1.3)
plt.plot([npix/2-pixCutEdge, npix/2-pixCutEdge],[0, (npix/2-pixCutEdge)*2-1],'w--')
plt.plot([0, (npix/2-pixCutEdge)*2-1],[npix/2-pixCutEdge, npix/2-pixCutEdge],'w--')
plt.annotate('IDW', [0,(npix/2-pixCutEdge)*2-5], c='w', size=15)
cticks=[-7, -6, -5, -4, -3, -2, -1]
cbar = plt.colorbar(ticks=cticks)
cbar.ax.set_yticklabels(['$10^{-7}$', '$10^{-6}$', '$10^{-5}$','$10^{-4}$','$10^{-3}$','$10^{-2}$', '$10^{-1}$'])
print(img1.min(), img1.max())
ax = plt.subplot(1,3,3)
plt.imshow(imgX, origin='lower', vmin =-3, vmax =np.log10(3e-1))
plt.plot([npix/2-pixCutEdge, npix/2-pixCutEdge],[0, (npix/2-pixCutEdge)*2-1],'w--')
plt.plot([0, (npix/2-pixCutEdge)*2-1],[npix/2-pixCutEdge, npix/2-pixCutEdge],'w--')
#plt.annotate('(IDW-ORG)/ORG', [0,(npix/2-pixCutEdge)*2-5], c='w', size=15)
cticks=[-5, -4, -3, -2, -1]
cbar = plt.colorbar(ticks=cticks)
cbar.ax.set_yticklabels(['$10^{-5}$','$10^{-4}$','$10^{-3}$','$10^{-2}$', '$10^{-1}$'])
print(np.max((psfMatORG-psfMatIDW)))
plt.savefig('figs/psfResidual_iccd{:}.pdf'.format(iccd))
def test_psfEllPlot(OVERPLOT=False):
#if ThisTask == 0:
if True:
prefix = 'psfEll20'
iccd = 1
iwave= 1
data = np.load('data/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
psf_e2 = data[3]
print(np.shape(imx))
npsf = np.shape(imx)[0]
plt.cla()
plt.close("all")
fig = plt.figure(figsize=(12,12))
plt.plot(imx, imy, 'r.')
plt.savefig('figs/psfPos.pdf')
#######
fig = plt.figure(figsize=(12, 12))
plt.subplots_adjust(wspace=0.1, hspace=0.1)
ax = plt.subplot(1, 1, 1)
for ipsf in range(npsf):
plt.plot(imx[ipsf], imy[ipsf], 'b.')
ang = np.arctan2(psf_e2[ipsf], psf_e1[ipsf])/2
ell = np.sqrt(psf_e1[ipsf]**2 + psf_e2[ipsf]**2)
ell *= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
plt.plot([imx[ipsf]-lcos, imx[ipsf]+lcos],[imy[ipsf]-lsin, imy[ipsf]+lsin],'b', lw=2)
###########
ang = 0.
ell = 0.05
ell*= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
#plt.plot([imx[898]-lcos, imx[898]+lcos],[imy[898]+5.-lsin, imy[898]+5.+lsin],'k', lw=2)
#plt.annotate('{:}'.format(ell/15), (imx[898]-2., imy[898]+6.), xycoords='data', fontsize=10)
plt.xlabel('CCD X (mm)')
plt.ylabel('CCD Y (mm)')
if OVERPLOT == True:
prefix = 'psfEll20IDW'
data = np.load('data/'+prefix+'_1_1.npy')
#imx= data[0]
#imy= data[1]
psf_e1 = data[0]
psf_e2 = data[1]
npsf = np.shape(imx)[0]
for ipsf in range(npsf):
#plt.plot(imx[ipsf], imy[ipsf], 'r.')
ang = np.arctan2(psf_e2[ipsf], psf_e1[ipsf])/2
ell = np.sqrt(psf_e1[ipsf]**2 + psf_e2[ipsf]**2)
ell *= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
plt.plot([imx[ipsf]-lcos, imx[ipsf]+lcos],[imy[ipsf]-lsin, imy[ipsf]+lsin],'r', lw=1)
plt.gca().set_aspect(1)
if OVERPLOT == True:
prefix = 'psfEllOPIDW'
plt.savefig('figs/'+prefix+'_iccd{:}.pdf'.format(iccd))
'''
def test_psfdEllPlot():
if ThisTask == 0:
prefix = 'psfEll20'
iccd = 1
iwave= 1
data = np.load('data/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
psf_e2 = data[3]
psf_sz = data[4]
print(np.shape(imx))
npsf = np.shape(imx)[0]
ellX = np.sqrt(psf_e1**2 + psf_e2**2)
angX = np.arctan2(psf_e2, psf_e1)/2
angX = np.rad2deg(angX)
szX = psf_sz
##############################
prefix = 'psfEll20IDW'
data = np.load('data/'+prefix+'_1_1.npy')
#imx= data[0]
#imy= data[1]
psf_e1 = data[0]
psf_e2 = data[1]
psf_sz = data[2]
ellY = np.sqrt(psf_e1**2 + psf_e2**2)
angY = np.arctan2(psf_e2, psf_e1)/2
angY = np.rad2deg(angY)
szY = psf_sz
##############################
fig=plt.figure(figsize=(15, 4))
ax = plt.subplot(1,3,1)
plt.hist(ellX, bins=20, color='b', alpha=0.5)
plt.hist(ellY, bins=20, color='r', alpha=0.5)
plt.xlabel('$\epsilon$')
plt.ylabel('PDF')
ax = plt.subplot(1,3,2)
plt.hist((ellY-ellX)/ellX, bins=20, color='r', alpha=0.5)
plt.xlabel('$(\epsilon_{\\rm IDW}-\epsilon_{\\rm ORG})/\epsilon_{\\rm ORG}$')
plt.ylabel('PDF')
ax = plt.subplot(1,3,3)
plt.hist((angY-angX)/angX, bins=20, color='r', alpha=0.5, range=[-0.1, 0.1])
plt.xlabel('$(\\alpha_{\\rm IDW}-\\alpha_{\\rm ORG})/\\alpha_{\\rm ORG}$')
plt.ylabel('PDF')
plt.savefig('figs/psfEllOPIDWPDF.pdf')
fig=plt.figure(figsize=(4, 4))
plt.hist((szY-szX)/szX, bins=20, color='r', alpha=0.5)
plt.xlabel('$(R_{\\rm IDW}-R_{\\rm ORG})/R_{\\rm ORG}$')
plt.ylabel('PDF')
plt.savefig('figs/psfEllOPIDWPDF_dsz.pdf')
'''
def test_psfdEllabsPlot(iccd):
#if ThisTask == 0:
if True:
prefix = 'psfEll20'
#iccd = 1
#iwave= 1
data = np.load('data/'+prefix+'_{:}_1.npy'.format(iccd))
imx= data[0]
imy= data[1]
psf_e1 = data[2]
psf_e2 = data[3]
psf_sz = data[4]
print(np.shape(imx))
npsf = np.shape(imx)[0]
ellX = np.sqrt(psf_e1**2 + psf_e2**2)
angX = np.arctan2(psf_e2, psf_e1)/2
angX = np.rad2deg(angX)
szX = psf_sz
##############################
prefix = 'psfEll20IDW'
data = np.load('data/'+prefix+'_{:}_1.npy'.format(iccd))
#imx= data[0]
#imy= data[1]
psf_e1 = data[0]
psf_e2 = data[1]
psf_sz = data[2]
ellY = np.sqrt(psf_e1**2 + psf_e2**2)
angY = np.arctan2(psf_e2, psf_e1)/2
angY = np.rad2deg(angY)
szY = psf_sz
##############################
fig=plt.figure(figsize=(6, 5))
grid = plt.GridSpec(3,1,left=0.15, right=0.95, wspace=None, hspace=0.02)
#plt.subplots_adjust(left=None,bottom=None,right=None,top=None,wspace=None,hspace=0.02)
ax = plt.subplot(grid[0:2,0])
plt.plot([0.01,0.1],[0.01,0.1], 'k--', lw=1. )
plt.scatter(ellX, ellY, color='b', alpha=1., s=3., edgecolors='None')
plt.xlim([0.015, 0.085])
plt.ylim([0.015, 0.085])
plt.ylabel('$\epsilon_{\\rm IDW}$')
plt.gca().axes.get_xaxis().set_visible(False)
ax = plt.subplot(grid[2,0])
plt.plot([0.015,0.085],[0.,0.], 'k--', lw=1. )
plt.scatter(ellX, (ellY-ellX), color='b', s=3., edgecolors='None')
plt.xlim([0.015, 0.085])
plt.ylim([-0.0018, 0.0018])
plt.xlabel('$\epsilon_{\\rm ORG}$')
plt.ylabel('$\Delta$')
plt.savefig('figs/psfEllOPIDWPDF_{:}.pdf'.format(iccd))
fig=plt.figure(figsize=(4, 4))
plt.hist((szY-szX)/szX, bins=20, color='r', alpha=0.5)
plt.xlabel('$(R_{\\rm IDW}-R_{\\rm ORG})/R_{\\rm ORG}$')
plt.ylabel('PDF')
plt.savefig('figs/psfEllOPIDWPDF_dsz_{:}.pdf'.format(iccd))
class PSFMatsIDW_coverage(unittest.TestCase):
def test_psfIDW_(self):
#comm = MPI.COMM_WORLD
#ThisTask = comm.Get_rank()
#NTasks = comm.Get_size()
iccd = 1
iwave= 1
ipsf = 400
psfPath = '/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/CSSOS_psf_20210326/CSST_psf_ciomp'
#test_psfIDW(iccd, iwave, psfPath, ThisTask, NTasks)
test_psfIDW(iccd, iwave, psfPath)
'''
for iccd in range(7, 10):
res = np.zeros(400)
for ipsf in range(1,401):
print(ipsf, end="\r")
res[ipsf-1] = test_psfResidualCalc(iccd, iwave, ipsf, psfPath)
#fig = plt.figure(figsize=(6,6))
#plt.hist(np.abs(res), bins=50)
#plt.xlim([0,1])
#plt.savefig('figs/psfResidualREE80PDF.pdf')
print("{:}:".format(iccd), res[res<=0.01].size/400*100)
'''
test_psfResidualPlot(iccd, iwave, ipsf, psfPath)
#test_psfEll(iccd, iwave, ThisTask, NTasks)
test_psfEll(iccd, iwave)
test_psfEllPlot(OVERPLOT=True)
#test_psfdEllPlot()
test_psfdEllabsPlot(iccd)
##############################
##############################
##############################
if __name__=='__main__':
unittest.main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment