Commit 0697a198 authored by Fang Yuedong's avatar Fang Yuedong
Browse files

v0 for testing

parent e1458fd3
from .PSFModel import PSFModel from .PSFModel import PSFModel
from .PSFGauss import PSFGauss from .PSFGauss import PSFGauss
from .PSFInterp import PSFInterp from .PSFInterp.PSFInterp import PSFInterp
from .FieldDistortion import FieldDistortion from .FieldDistortion import FieldDistortion
\ No newline at end of file
import numpy as np import numpy as np
import os import os
from datetime import datetime from datetime import datetime
import argparse
def parse_args():
'''
Parse command line arguments. Many of the following
can be set in the .yaml config file as well.
'''
parser = argparse.ArgumentParser()
parser.add_argument('config_file', help='.yaml config file for simulation settings.')
parser.add_argument('-c', '--config_dir', help='Directory that houses the ,yaml config file.')
parser.add_argument('-d', '--data_dir', help='Directory that houses the input data.')
parser.add_argument('-w', '--work_dir', help='The path for output.')
return parser.parse_args()
def imgName(tt=0): def imgName(tt=0):
ut = datetime.utcnow() ut = datetime.utcnow()
...@@ -23,45 +36,54 @@ def imgName(tt=0): ...@@ -23,45 +36,54 @@ def imgName(tt=0):
return namekey return namekey
def makeSubDir(path_dict, config): # def makeSubDir(path_dict, config):
subImgdir = path_dict["output_img_dir"] + config["mockImgDir"] + '/' # subImgdir = path_dict["output_img_dir"] + config["mockImgDir"] + '/'
if not os.path.exists(subImgdir): # if not os.path.exists(subImgdir):
os.system("mkdir %s"%subImgdir) # os.system("mkdir %s"%subImgdir)
imgKey0 = imgName(tt=0) # imgKey0 = imgName(tt=0)
imgKey1 = imgName(tt=1) # imgKey1 = imgName(tt=1)
imgKey2 = imgName(tt=2) # imgKey2 = imgName(tt=2)
subImgdir = subImgdir + imgKey0 + config["reIndex"] + config["reShear"] + "/" # subImgdir = subImgdir + imgKey0 + config["reIndex"] + config["reShear"] + "/"
if not os.path.exists(subImgdir): # if not os.path.exists(subImgdir):
os.system("mkdir %s"%subImgdir) # os.system("mkdir %s"%subImgdir)
return subImgdir, imgKey0, imgKey1, imgKey2 # return subImgdir, imgKey0, imgKey1, imgKey2
def makeSubDir_PointingList(path_dict, config, pointing_ID=0): def makeSubDir_PointingList(path_dict, config, pointing_ID=0):
imgDir = path_dict["work_dir"] + config["mockImgDir"] + '/' imgDir = os.path.join(path_dict["work_dir"], config["run_name"])
if not os.path.exists(imgDir): if not os.path.exists(imgDir):
os.system("mkdir %s"%imgDir) try:
# prefix = "MSC_" + config["date_obs"] + config["time_obs"] + "_" + str(pointing_ID).rjust(7, '0') os.makedirs(imgDir, exist_ok=True)
except OSError:
pass
prefix = "MSC_" + str(pointing_ID).rjust(7, '0') prefix = "MSC_" + str(pointing_ID).rjust(7, '0')
subImgdir = os.path.join(imgDir, prefix) subImgdir = os.path.join(imgDir, prefix)
if not os.path.exists(subImgdir): if not os.path.exists(subImgdir):
os.system("mkdir %s"%subImgdir) try:
os.makedirs(subImgdir, exist_ok=True)
except OSError:
pass
return subImgdir, prefix return subImgdir, prefix
def getShearFiled(config, shear_cat_file=None): def getShearFiled(config, shear_cat_file=None):
if not config["shear_method"] in ["constant", "extra"]: if not config["shear_setting"]["shear_type"] in ["constant", "extra"]:
raise ValueError("Please set a right 'shear_method' parameter.") raise ValueError("Please set a right 'shear_method' parameter.")
if config["shear_method"] == "constant": if config["shear_setting"]["shear_type"] == "constant":
g1 = config["reduced_g1"] g1 = config["shear_setting"]["reduced_g1"]
g2 = config["reduced_g2"] g2 = config["shear_setting"]["reduced_g2"]
reduced_shear = np.sqrt(g1**2 + g2**2) reduced_shear = np.sqrt(g1**2 + g2**2)
nshear = 1 nshear = 1
# TODO logging # TODO logging
else: else:
# TODO logging # TODO logging
# if not os.path.exists(shear_cat_file): if not os.path.exists(shear_cat_file):
# raise ValueError("Cannot find shear catalog file.") raise ValueError("Cannot find shear catalog file.")
# shearCat = np.loadtxt(shear_cat_file) try:
# nshear = shearCat.shape[0] shearCat = np.loadtxt(shear_cat_file)
# g1, g2 = shearCat[:, 0], shearCat[:, 1] nshear = shearCat.shape[0]
pass g1, g2 = shearCat[:, 0], shearCat[:, 1]
except:
print("Failed to the shear catalog file.")
print("Setting to no shear.")
g1, g2 = 0., 0.
return g1, g2, nshear return g1, g2, nshear
\ No newline at end of file
...@@ -12,11 +12,12 @@ ...@@ -12,11 +12,12 @@
#PBS -q batch #PBS -q batch
#PBS -u fangyuedong #PBS -u fangyuedong
NP=54 NP=40
date date
echo $NP echo $NP
# mpirun -np $NP --oversubscribe -H comput101 python /public/home/fangyuedong/CSST/ObservationSim/runExposure.py # mpirun -np $NP --oversubscribe -H comput101 python /public/home/fangyuedong/CSST/ObservationSim/runExposure.py
# python /public/home/fangyuedong/sim_code_release/CSSTSim/ObservationSim/preprocess.py
# mpirun -np $NP python /public/home/fangyuedong/sim_code_release/CSSTSim/ObservationSim/runExposure.py
python /public/home/fangyuedong/test/CSST/ObservationSim/preprocess.py mpirun -np $NP python /public/home/fangyuedong/sim_code_release/CSSTSim/ObservationSim/runExposure.py config_sim.yaml -c /public/home/fangyuedong/sim_code_release/CSSTSim/config
mpirun -np $NP python /public/home/fangyuedong/test/CSST/ObservationSim/runExposure.py
from ObservationSim import Observation from ObservationSim import Observation
from Pointing import * from _util import parse_args
from datetime import datetime
import os import os
import numpy as np import numpy as np
import galsim import galsim
import yaml
import gc import gc
gc.enable() gc.enable()
def Pointing(config, pointing_filename, data_dir):
pointing_file = os.path.join(data_dir, pointing_filename)
f = open(pointing_file, 'r')
for _ in range(1):
header = f.readline()
iline = 0
pRA = []
pDEC = []
for line in f:
line = line.strip()
columns = line.split()
pRA.append(float(columns[0]))
pDEC.append(float(columns[1]))
f.close()
pRA = np.array(pRA)
pDEC = np.array(pDEC)
# Create calibration pointings
# NOTE: temporary implementation
ncal = config['obs_setting']['np_cal']
pointing_type = ['MS']*len(pRA)
pRA = np.append([pRA[0]]*ncal, pRA)
pDEC = np.append([pDEC[0]]*ncal, pDEC)
pointing_type = ['CAL']*ncal + pointing_type
# Calculate starting time(s)
# NOTE: temporary implementation
t0 = datetime(2021, 5, 25, 12, 0, 0)
t = datetime.timestamp(t0)
timestamp_obs = []
delta_t = 10 # Time elapsed between exposures (minutes)
for i in range(len(pointing_type)):
timestamp_obs.append(t)
if pointing_type[i] == 'CAL':
t += 3 * delta_t * 60 # 3 calibration exposure
elif pointing_type[i] == 'MS':
t += delta_t * 60
timestamp_obs = np.array(timestamp_obs)
pointing_type = np.array(pointing_type)
return pRA, pDEC, timestamp_obs, pointing_type
def MakeDirectories(work_dir, run_name, nPointings, pRange=None):
if not os.path.exists(work_dir):
try:
os.makedirs(work_dir, exist_ok=True)
except OSError:
pass
imgDir = os.path.join(work_dir, run_name)
if not os.path.exists(imgDir):
try:
os.makedirs(imgDir, exist_ok=True)
except OSError:
pass
prefix = "MSC_"
for pointing_ID in range(nPointings):
if pRange is not None:
if pointing_ID not in pRange:
continue
fname=prefix + str(pointing_ID).rjust(7, '0')
subImgDir = os.path.join(imgDir, fname)
if not os.path.exists(subImgDir):
try:
os.makedirs(subImgDir, exist_ok=True)
except OSError:
pass
def runSim():
"""
Main simulation call.
"""
args = parse_args()
if args.config_dir is None:
args.config_dir = ''
args.config_dir = os.path.abspath(args.config_dir)
args.config_file = os.path.join(args.config_dir, args.config_file)
with open(args.config_file, "r") as stream:
try:
config = yaml.safe_load(stream)
for key, value in config.items():
print (key + " : " + str(value))
except yaml.YAMLError as exc:
print(exc)
config["obs_setting"]["image_rot"] = float(config["obs_setting"]["image_rot"])*galsim.degrees
if args.data_dir is not None:
config['data_dir'] = args.data_dir
if args.work_dir is not None:
config['work_dir'] = args.work_dir
pRA, pDEC, timestamp_obs, pointing_type = Pointing(config=config, pointing_filename=config['pointing_file'], data_dir=config['data_dir'])
MakeDirectories(work_dir=config['work_dir'], run_name=config['run_name'], nPointings=len(pRA), pRange=config['obs_setting']['run_pointings'])
if "run_chips" in config["obs_setting"]:
run_chips = config["obs_setting"]["run_chips"]
else:
run_chips = None
# from Config import ConfigDir
# path_dict = ConfigDir(config=config, work_dir=config["work_dir"], data_dir=config["data_dir"])
# for key, value in path_dict.items():
# print (key + " : " + str(value))
obs = Observation(config=config, work_dir=config['work_dir'], data_dir=config['data_dir'])
if config["pointing_file"] is None:
obs.runExposure(chips=run_chips)
else:
obs.runExposure_MPI_PointingList(
ra_cen=pRA,
dec_cen=pDEC,
pRange=config["obs_setting"]["run_pointings"],
timestamp_obs=timestamp_obs,
pointing_type=pointing_type,
exptime=config["obs_setting"]["exp_time"],
use_mpi=config["run_option"]["use_mpi"],
chips=run_chips
)
print("run finished")
if __name__=='__main__':
runSim()
############################################# #############################################
# Testing run one exposure (NOT using MPI) # Testing run one exposure (NOT using MPI)
# ipoint = 2 # ipoint = 2
...@@ -18,6 +141,6 @@ gc.enable() ...@@ -18,6 +141,6 @@ gc.enable()
############################################# #############################################
# Testing run pointing list (using MPI) # Testing run pointing list (using MPI)
obs = Observation(work_dir=work_dir, data_dir=data_dir) # obs = Observation(work_dir=work_dir, data_dir=data_dir)
# obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type) # # obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type)
obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=exp_time) # obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=exp_time)
---
###############################################
#
# Configuration file for CSST simulation
# CSST-Sim Group, 2021/10/07, version 0.3
#
###############################################
# Base diretories and naming setup
# Can add some of the command-line arguments here as well;
# OK to pass either way or both, as long as they are consistent
# work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/"
work_dir: "/public/home/fangyuedong/sim_code_release/CSSTSim/workplace/"
data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
run_name: "TEST"
# (Optional) a file of point list
pointing_file: "pointing10_20210202.dat"
# Whether to use MPI
run_option:
use_mpi: YES
n_cores: 40
# Output catalog only?
# If yes, no imaging simulation will run
out_cat_only: NO
# Only simulate stars?
star_only: NO
# Only simulate galaxies?
galaxy_only: NO
###############################################
# Observation setting
###############################################
obs_setting:
# Options for survey types:
# "Photometric": simulate photometric chips only
# "Spectroscoplic": simulate slitless spectroscopic chips only
# "All": simulate full focal plane
survey_type: "Photometric"
# Exposure time [seconds]
exp_time: 150.
# Observation starting date & time
# (Subject to change)
date_obs: "210525" # [yymmdd]
time_obs: "120000" # [hhmmss]
# Default Pointing [degrees]
# Note: NOT valid when a pointing list file is specified
ra_center: 60.
dec_center: -40.
# Image rotation [degree]
image_rot: -113.4333
# Number of calibration pointings
# Note: only valid when a pointing list is specified
np_cal: 1
# (Optional) only run specific pointing(s).
# Note: only valid when a pointing list is specified
run_pointings: [0, 1, 2, 3, 4]
# (Optional) only run specific chip(s)
# Note: for all pointings
run_chips: [6, 25]
###############################################
# Input path setting
###############################################
# Default path settings for WIDE survey simulation
input_path:
cat_dir: "Catalog_20210126/"
star_cat: "MMW_Cluster_D20_healpix_21.hdf5"
galaxy_cat: "galaxyCats_r_10.0_healpix.hdf5"
SED_templates_path:
star_SED: "Catalog_20210126/SpecLib.hdf5"
galaxy_SED: "Templates/Galaxy/"
# TODO: should the following path settings be made hidden from user?
Efficiency_curve_path:
filter_eff: "Filters/"
ccd_eff: "Filter_CCD_Mirror/ccd/"
mirror_eff: "Filter_CCD_Mirror/mirror_ccdnote.txt"
CR_data_path: "CRdata/"
sky_data_path: "skybackground/sky_emiss_hubble_50_50_A.dat"
SLS_path:
SLS_conf: "CONF"
SLS_norm: "normalize_filter"
###############################################
# PSF setting
###############################################
psf_setting:
# Which PSF model to use:
# "Gauss": simple gaussian profile
# "Interp": Interpolated PSF from sampled ray-tracing data
psf_model: "Interp"
# PSF size [arcseconds]
# radius of 80% energy encircled
# NOTE: only valid for "Gauss" PSF
psf_rcont: 0.15
# path to PSF data
# NOTE: only valid for "Interp" PSF
psf_dir: "csstPSFdata/CSSOS_psf_20210108/CSST_psf_ciomp_2p5um_cycle3_ccr90_proc"
# sigma_spin: 0.0 # psf spin?
###############################################
# Shear setting
###############################################
shear_setting:
# Options to generate mock shear field:
# "constant": all galaxies are assigned a constant reduced shear
# "catalog": from catalog (not available yet)
# "extra": from seprate file
shear_type: "constant"
# For constant shear filed
reduced_g1: 0.026
reduced_g2: 0.015
# Representation of the shear vector?
reShear: "E"
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
# Extra shear catalog
# (currently not used)
# shear_cat: "mockShear.cat"
###############################################
# Instrumental effects setting
###############################################
ins_effects:
# switches
field_dist: ON # Whether to add field distortions
add_back: ON # Whether to add sky background
add_dark: ON # Whether to add dark noise
add_readout: ON # Whether to add read-out (Gaussian) noise
add_bias: ON # Whether to add bias-level to images
shutter_effect: ON # Whether to add shutter effect
flat_fielding: ON # Whether to add flat-fielding effect
prnu_effect: ON # Whether to add PRNU effect
non_linear: OFF # Whether to add non-linearity
cosmic_ray: ON # Whether to add cosmic-ray
cray_differ: ON # Whether to generate different cosmic ray maps CAL and MS output
cte_trail: ON # Whether to simulate CTE trails
saturbloom: ON # Whether to simulate Saturation & Blooming
add_badcolumns: ON # Whether to add bad columns
add_hotpixels: ON # Whether to add hot pixels
add_deadpixels: ON # Whether to add dead(dark) pixels
bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect
# values
dark_exptime: 300 # Exposure time for dark current frames [seconds]
flat_exptime: 150 # Exposure time for flat-fielding frames [seconds]
readout_time: 40 # The read-out time for each channel [seconds]
df_strength: 2.3 # Sillicon sensor diffusion strength
bias_level: 500 # bias level [e-/pixel]
gain: 1.1 # Gain
full_well: 90000 # Full well depth [e-]
NBias: 1 # Number of bias frames to be exported for each exposure
NDark: 1 # Number of dark frames to be exported for each exposure
NFlat: 1 # Number of flat frames to be exported for each exposure
###############################################
# Output options
###############################################
output_setting:
readout16: OFF # Whether to export as 16 channels (subimages) with pre- and over-scan
shutter_output: OFF # Whether to export shutter effect 16-bit image
bias_output: ON # Whether to export bias frames
dark_output: ON # Whether to export the combined dark current files
flat_output: ON # Whether to export the combined flat-fielding files
prnu_output: OFF # Whether to export the PRNU (pixel-to-pixel flat-fielding) files
###############################################
# Random seeds
###############################################
random_seeds:
seed_Av: 121212 # Seed for generating random intrinsic extinction
seed_poisson: 20210601 # Seed for Poisson noise
seed_CR: 20210317 # Seed for generating random cosmic ray maps
seed_flat: 20210101 # Seed for generating random flat fields
seed_prnu: 20210102 # Seed for photo-response non-uniformity
seed_gainNonUniform: 20210202 # Seed for gain nonuniformity
seed_biasNonUniform: 20210203 # Seed for bias nonuniformity
seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity
seed_badcolumns: 20240309 # Seed for bad columns
seed_defective: 20210304 # Seed for defective (bad) pixels
seed_readout: 20210601 # Seed for read-out gaussian noise
...
\ No newline at end of file
import yaml
if __name__ == '__main__':
stream = open("config_sim.yaml", 'r')
# dictionary = yaml.load(stream)
dictionary = yaml.load(stream, Loader=yaml.SafeLoader)
# print(dictionary)
for key, value in dictionary.items():
print (key + " : " + str(value))
with open("config_sim.yaml", "r") as stream:
try:
dictionary = yaml.safe_load(stream)
for key, value in dictionary.items():
print (key + " : " + str(value))
except yaml.YAMLError as exc:
print(exc)
\ No newline at end of file
from setuptools import setup, find_packages
setup(name='CSSTSim',
version='0.3',
packages=find_packages(),
# install_requires=[
# 'numpy>=1.18.5',
# 'galsim>=2.2.4',
# 'yaml>=5.3.1',
# 'astropy>=4.0.1',
# 'scipy>=1.5.0',
# 'mpi4py>=3.0.3',
# 'sep>=1.0.3',
# 'healpy>=1.14.0',
# 'h5py>=2.10.0',
# 'Cython>=0.29.21'
# 'numba>=0.50.1'
# ]
)
\ No newline at end of file
}q(U collectorqUcoverage v3.6b3qUlinesq}u.
\ No newline at end of file
import unittest
import sys,os,math
from itertools import islice
#import mpi4py.MPI as MPI
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
mpl.use('Agg')
import scipy.io
from scipy import ndimage
#sys.path.append("/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_v4_20210326")
sys.path.append("../")
from ObservationSim.PSF.PSFInterp import PSFConfig as myConfig
#import PSFUtil as myUtil
NPSF = 400
##############################
###计算PSF椭率###
def psfSecondMoments(psfMat, cenX, cenY, pixSize=1):
apr = 0.5 #arcsec, 0.5角秒内测量
fl = 28. #meters
pxs = 2.5 #microns
apr = np.deg2rad(apr/3600.)*fl*1e6
apr = apr/pxs
apr = np.int(np.ceil(apr))
I = psfMat
ncol = I.shape[1]
nrow = I.shape[0]
w = 0.0
w11 = 0.0
w12 = 0.0
w22 = 0.0
for icol in range(ncol):
for jrow in range(nrow):
x = icol*pixSize - cenX
y = jrow*pixSize - cenY
rr = np.sqrt(x*x + y*y)
wgt= 0.0
if rr <= apr:
wgt = 1.0
w += I[jrow, icol]*wgt
w11 += x*x*I[jrow, icol]*wgt
w12 += x*y*I[jrow, icol]*wgt
w22 += y*y*I[jrow, icol]*wgt
w11 /= w
w12 /= w
w22 /= w
sz = w11 + w22
e1 = (w11 - w22)/sz
e2 = 2.0*w12/sz
return sz, e1, e2
##############################
##############################
'''
def assignTasks(npsf, NTasks, ThisTask):
npsfPerTasks = int(npsf/NTasks)
iStart= 0 + npsfPerTasks*ThisTask
iEnd = npsfPerTasks + npsfPerTasks*ThisTask
if ThisTask == NTasks:
iEnd = npsf
return iStart, iEnd
'''
#def test_psfEll(iccd, iwave, psfPath, ThisTask, NTasks):
def test_psfEll(iccd, iwave, psfPath):
nccd = 30
npsf = NPSF
#iStart, iEnd = assignTasks(npsf, NTasks, ThisTask)
imx = np.zeros(npsf)
imy = np.zeros(npsf)
psf_e1 = np.zeros(npsf)
psf_e2 = np.zeros(npsf)
psf_sz = np.zeros(npsf)
#for ipsf in range(iStart+1, iEnd+1):
for ipsf in range(1, 401):
if ipsf != 1:
continue
print('ipsf-{:}'.format(ipsf), end='\r')
psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPath, InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
imx[ipsf-1] = psfInfo['image_x']+psfInfo['centroid_x']
imy[ipsf-1] = psfInfo['image_y']+psfInfo['centroid_y']
psfMat = psfInfo['psfMat']
cenX = 256
cenY = 256
sz, e1, e2 = psfSecondMoments(psfMat, cenX, cenY, pixSize=1)
psf_e1[ipsf-1] = e1
psf_e2[ipsf-1] = e2
psf_sz[ipsf-1] = sz
print('test:' ,sz, e1, e2)
#######
#comm.barrier()
#imx = comm.allreduce(imx, op=MPI.SUM)
#imy = comm.allreduce(imy, op=MPI.SUM)
#psf_e1 = comm.allreduce(psf_e1, op=MPI.SUM)
#psf_e2 = comm.allreduce(psf_e2, op=MPI.SUM)
#psf_sz = comm.allreduce(psf_sz, op=MPI.SUM)
#comm.barrier()
#if ThisTask == 0:
# arr = [imx, imy, psf_e1, psf_e2, psf_sz]
# np.save('/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_v4_20210326/test4report/data/psfEll20_{:}_{:}'.format(iccd, iwave), arr)
def test_psfEllPlot(OVERPLOT=False):
#if ThisTask == 0:
if True:
prefix = 'psfEll30'
iccd = 1
iwave= 1
data = np.load('data/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
psf_e2 = data[3]
print(np.shape(imx))
npsf = np.shape(imx)[0]
plt.cla()
plt.close("all")
fig = plt.figure(figsize=(12,12))
plt.plot(imx, imy, 'r.')
plt.savefig('figs/psfPos.pdf')
#######
fig = plt.figure(figsize=(12, 12))
plt.subplots_adjust(wspace=0.1, hspace=0.1)
ax = plt.subplot(1, 1, 1)
for ipsf in range(npsf):
plt.plot(imx[ipsf], imy[ipsf], 'r.')
ang = np.arctan2(psf_e2[ipsf], psf_e1[ipsf])/2
ell = np.sqrt(psf_e1[ipsf]**2 + psf_e2[ipsf]**2)
ell *= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
plt.plot([imx[ipsf]-lcos, imx[ipsf]+lcos],[imy[ipsf]-lsin, imy[ipsf]+lsin],'r', lw=2)
###########
ang = 0.
ell = 0.05
ell*= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
plt.plot([imx[898]-lcos, imx[898]+lcos],[imy[898]+5.-lsin, imy[898]+5.+lsin],'k', lw=2)
plt.annotate('{:}'.format(ell/15), (imx[898]-2., imy[898]+6.), xycoords='data', fontsize=10)
plt.xlabel('CCD X (mm)')
plt.ylabel('CCD Y (mm)')
if OVERPLOT == True:
prefix = 'psfEll20'
data = np.load('data/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
psf_e2 = data[3]
npsf = np.shape(imx)[0]
for ipsf in range(npsf):
plt.plot(imx[ipsf], imy[ipsf], 'b.')
ang = np.arctan2(psf_e2[ipsf], psf_e1[ipsf])/2
ell = np.sqrt(psf_e1[ipsf]**2 + psf_e2[ipsf]**2)
ell *= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
plt.plot([imx[ipsf]-lcos, imx[ipsf]+lcos],[imy[ipsf]-lsin, imy[ipsf]+lsin],'b', lw=2)
plt.gca().set_aspect(1)
if OVERPLOT == True:
prefix = 'psfEllOP'
plt.savefig('figs/'+prefix+'_iccd{:}.pdf'.format(iccd))
class PSFMatsEll_coverage(unittest.TestCase):
def test_psfEll_(self):
#comm = MPI.COMM_WORLD
#ThisTask = comm.Get_rank()
#NTasks = comm.Get_size()
print('#####haha#####')
iccd = 1
iwave= 1
psfPath = '/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/CSSOS_psf_20210326/CSST_psf_ciomp_20x20field'
#test_psfEll(iccd, iwave, psfPath, ThisTask, NTasks)
test_psfEll(iccd, iwave, psfPath)
test_psfEllPlot(OVERPLOT=True)
##############################
##############################
##############################
if __name__=='__main__':
unittest.main()
import unittest
import sys,os,math
from itertools import islice
import mpi4py.MPI as MPI
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
mpl.use('Agg')
import scipy.io
from scipy import ndimage
#sys.path.append("/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_v4_20210326")
# sys.path.append("../")
from ObservationSim.PSF.PSFInterp import PSFConfig as myConfig
from ObservationSim.PSF.PSFInterp import PSFUtil as myUtil
NPSF = 400
##############################
###计算PSF椭率###
def psfSecondMoments(psfMat, cenX, cenY, pixSize=1):
apr = 0.5 #arcsec, 0.5角秒内测量
fl = 28. #meters
pxs = 2.5 #microns
apr = np.deg2rad(apr/3600.)*fl*1e6
apr = apr/pxs
apr = np.int(np.ceil(apr))
I = psfMat
ncol = I.shape[1]
nrow = I.shape[0]
w = 0.0
w11 = 0.0
w12 = 0.0
w22 = 0.0
for icol in range(ncol):
for jrow in range(nrow):
x = icol*pixSize - cenX
y = jrow*pixSize - cenY
rr = np.sqrt(x*x + y*y)
wgt= 0.0
if rr <= apr:
wgt = 1.0
w += I[jrow, icol]*wgt
w11 += x*x*I[jrow, icol]*wgt
w12 += x*y*I[jrow, icol]*wgt
w22 += y*y*I[jrow, icol]*wgt
w11 /= w
w12 /= w
w22 /= w
sz = w11 + w22
e1 = (w11 - w22)/sz
e2 = 2.0*w12/sz
return sz, e1, e2
##############################
##############################
'''
def assignTasks(npsf, NTasks, ThisTask):
npsfPerTasks = int(npsf/NTasks)
iStart= 0 + npsfPerTasks*ThisTask
iEnd = npsfPerTasks + npsfPerTasks*ThisTask
if ThisTask == NTasks:
iEnd = npsf
return iStart, iEnd
'''
#def test_psfIDW(iccd, iwave, psfPath, ThisTask, NTasks):
def test_psfIDW(iccd, iwave, psfPath):
nccd = 30
npsfA = 900
npsfB = 400
#iStart, iEnd = assignTasks(400, NTasks, ThisTask)
psfPathA = psfPath+'_30x30field'
psfPathB = psfPath+'_20x20field'
imxA = np.zeros(npsfA)
imyA = np.zeros(npsfA)
psfA = np.zeros([npsfA, 512, 512])
imxB = np.zeros(npsfB)
imyB = np.zeros(npsfB)
psfB = np.zeros([npsfB, 512, 512])
#for ipsf in range(iStart+1, iEnd+1):
for ipsf in range(1, npsfA+1):
print('ipsfA:', ipsf, end='\r', flush=True)
psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPathA, InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
imxA[ipsf-1] = psfInfo['image_x'] + psfInfo['centroid_x']
imyA[ipsf-1] = psfInfo['image_y'] + psfInfo['centroid_y']
psfA[ipsf-1, :, :] = psfInfo['psfMat']
for ipsf in range(1, npsfB+1):
print('ipsfB:', ipsf, end='\r', flush=True)
psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPathB, InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
imxB[ipsf-1] = psfInfo['image_x'] + psfInfo['centroid_x']
imyB[ipsf-1] = psfInfo['image_y'] + psfInfo['centroid_y']
psfB[ipsf-1, :, :] = psfInfo['psfMat']
#myConfig.psfMaker_IDW(px, py, PSFMat, cen_col, cen_row, IDWindex=2, OnlyNeighbors=True, hoc=None, hoclist=None, PSFCentroidWgt=False)
for ipsf in range(npsfB):
print('ipsf:', ipsf, end='\r', flush=True)
px = imxB[ipsf]
py = imyB[ipsf]
cen_col = imxA
cen_row = imyA
PSFMat = psfA
psfIDW = myConfig.psfMaker_IDW(px, py, PSFMat, cen_col, cen_row, IDWindex=2, OnlyNeighbors=True, hoc=None, hoclist=None, PSFCentroidWgt=False)
np.save('figs/psfIDW/psfIDW_{:}_{:}_{:}'.format(iccd, iwave, ipsf+1), psfIDW)
#def test_psfEll(iccd, iwave, ThisTask, NTasks):
def test_psfEll(iccd, iwave):
nccd = 30
npsf = 400
#iStart, iEnd = assignTasks(npsf, NTasks, ThisTask)
#imx = np.zeros(npsf)
#imy = np.zeros(npsf)
psf_e1 = np.zeros(npsf)
psf_e2 = np.zeros(npsf)
psf_sz = np.zeros(npsf)
#for ipsf in range(iStart+1, iEnd+1):
for ipsf in range(1, 401):
if ipsf > 1:
continue
print('ipsf-{:}'.format(ipsf), end='\r')
#psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPath, InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
#psfMat = psfInfo['psfMat']
#imx[ipsf-1] = psfInfo['image_x']+psfInfo['centroid_x']
#imy[ipsf-1] = psfInfo['image_y']+psfInfo['centroid_y']
psfMat = np.load('figs/psfIDW/psfIDW_{:}_{:}_{:}.npy'.format(iccd, iwave, ipsf)) #psfInfo['psfMat']
cenX = 256
cenY = 256
sz, e1, e2 = psfSecondMoments(psfMat, cenX, cenY, pixSize=1)
psf_e1[ipsf-1] = e1
psf_e2[ipsf-1] = e2
psf_sz[ipsf-1] = sz
#######
#comm.barrier()
#imx = comm.allreduce(imx, op=MPI.SUM)
#imy = comm.allreduce(imy, op=MPI.SUM)
#psf_e1 = comm.allreduce(psf_e1, op=MPI.SUM)
#psf_e2 = comm.allreduce(psf_e2, op=MPI.SUM)
#psf_sz = comm.allreduce(psf_sz, op=MPI.SUM)
#comm.barrier()
#if ThisTask == 0:
# arr = [psf_e1, psf_e2, psf_sz]
# np.save('/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_v4_20210326/test4report/data/psfEll20IDW_{:}_{:}'.format(iccd, iwave), arr)
'''
def test_psfResidualCalc(iccd, iwave, ipsf, psfPath):
psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPath+'_20x20field', InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
psfMatORG = psfInfo['psfMat']
psfMatIDW = np.load('/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_v4_20210326/test4report/figs/psfIDW/psfIDW_{:}_{:}_{:}.npy'.format(iccd, iwave, ipsf))
_,tREE80 = myUtil.psfEncircle(psfMatORG, fraction=0.8, psfSampleSizeInMicrons=2.5, focalLengthInMeters=28, cenPix=None)
tREE80_pix = np.int32(np.ceil(tREE80/(0.074/2/2))[0])
#print(tREE80, np.ceil(tREE80/(0.074/2/2)), tREE80_pix)
timg0 = psfMatORG[256-tREE80_pix:256+tREE80_pix, 256-tREE80_pix:256+tREE80_pix]
timg1 = psfMatIDW[256-tREE80_pix:256+tREE80_pix, 256-tREE80_pix:256+tREE80_pix]
#print("residual::", np.max((timg1-timg0)/timg0), np.min((timg1-timg0)/timg0), np.mean((timg1-timg0)/timg0))
return np.mean((timg1-timg0)/timg0)
'''
def test_psfResidualPlot(iccd, iwave, ipsf, psfPath):
psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPath+'_20x20field', InputMaxPixelPos=True, PSFCentroidWgt=True, VPSF=False)
psfMatORG = psfInfo['psfMat']
psfMatIDW = np.load('figs/psfIDW/psfIDW_{:}_{:}_{:}.npy'.format(iccd, iwave, ipsf))
npix = psfMatORG.shape[0]
pixCutEdge= int(npix/2-15)
img0 = psfMatORG[pixCutEdge:npix-pixCutEdge, pixCutEdge:npix-pixCutEdge]
img1 = psfMatIDW[pixCutEdge:npix-pixCutEdge, pixCutEdge:npix-pixCutEdge]
imgX = (img1 - img0)/img0
img0 = np.log10(img0)
img1 = np.log10(img1)
imgX = np.log10(np.abs(imgX))
fig = plt.figure(figsize=(18,4))
ax = plt.subplot(1,3,1)
plt.imshow(img0, origin='lower', vmin=-7, vmax=-1.3)
plt.plot([npix/2-pixCutEdge, npix/2-pixCutEdge],[0, (npix/2-pixCutEdge)*2-1],'w--')
plt.plot([0, (npix/2-pixCutEdge)*2-1],[npix/2-pixCutEdge, npix/2-pixCutEdge],'w--')
plt.annotate('ORG', [0,(npix/2-pixCutEdge)*2-5], c='w', size=15)
cticks=[-7, -6, -5, -4, -3, -2, -1]
cbar = plt.colorbar(ticks=cticks)
cbar.ax.set_yticklabels(['$10^{-7}$', '$10^{-6}$', '$10^{-5}$','$10^{-4}$','$10^{-3}$','$10^{-2}$', '$10^{-1}$'])
print(img0.min(), img0.max())
ax = plt.subplot(1,3,2)
plt.imshow(img1, origin='lower', vmin=-7, vmax=-1.3)
plt.plot([npix/2-pixCutEdge, npix/2-pixCutEdge],[0, (npix/2-pixCutEdge)*2-1],'w--')
plt.plot([0, (npix/2-pixCutEdge)*2-1],[npix/2-pixCutEdge, npix/2-pixCutEdge],'w--')
plt.annotate('IDW', [0,(npix/2-pixCutEdge)*2-5], c='w', size=15)
cticks=[-7, -6, -5, -4, -3, -2, -1]
cbar = plt.colorbar(ticks=cticks)
cbar.ax.set_yticklabels(['$10^{-7}$', '$10^{-6}$', '$10^{-5}$','$10^{-4}$','$10^{-3}$','$10^{-2}$', '$10^{-1}$'])
print(img1.min(), img1.max())
ax = plt.subplot(1,3,3)
plt.imshow(imgX, origin='lower', vmin =-3, vmax =np.log10(3e-1))
plt.plot([npix/2-pixCutEdge, npix/2-pixCutEdge],[0, (npix/2-pixCutEdge)*2-1],'w--')
plt.plot([0, (npix/2-pixCutEdge)*2-1],[npix/2-pixCutEdge, npix/2-pixCutEdge],'w--')
#plt.annotate('(IDW-ORG)/ORG', [0,(npix/2-pixCutEdge)*2-5], c='w', size=15)
cticks=[-5, -4, -3, -2, -1]
cbar = plt.colorbar(ticks=cticks)
cbar.ax.set_yticklabels(['$10^{-5}$','$10^{-4}$','$10^{-3}$','$10^{-2}$', '$10^{-1}$'])
print(np.max((psfMatORG-psfMatIDW)))
plt.savefig('figs/psfResidual_iccd{:}.pdf'.format(iccd))
def test_psfEllPlot(OVERPLOT=False):
#if ThisTask == 0:
if True:
prefix = 'psfEll20'
iccd = 1
iwave= 1
data = np.load('data/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
psf_e2 = data[3]
print(np.shape(imx))
npsf = np.shape(imx)[0]
plt.cla()
plt.close("all")
fig = plt.figure(figsize=(12,12))
plt.plot(imx, imy, 'r.')
plt.savefig('figs/psfPos.pdf')
#######
fig = plt.figure(figsize=(12, 12))
plt.subplots_adjust(wspace=0.1, hspace=0.1)
ax = plt.subplot(1, 1, 1)
for ipsf in range(npsf):
plt.plot(imx[ipsf], imy[ipsf], 'b.')
ang = np.arctan2(psf_e2[ipsf], psf_e1[ipsf])/2
ell = np.sqrt(psf_e1[ipsf]**2 + psf_e2[ipsf]**2)
ell *= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
plt.plot([imx[ipsf]-lcos, imx[ipsf]+lcos],[imy[ipsf]-lsin, imy[ipsf]+lsin],'b', lw=2)
###########
ang = 0.
ell = 0.05
ell*= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
#plt.plot([imx[898]-lcos, imx[898]+lcos],[imy[898]+5.-lsin, imy[898]+5.+lsin],'k', lw=2)
#plt.annotate('{:}'.format(ell/15), (imx[898]-2., imy[898]+6.), xycoords='data', fontsize=10)
plt.xlabel('CCD X (mm)')
plt.ylabel('CCD Y (mm)')
if OVERPLOT == True:
prefix = 'psfEll20IDW'
data = np.load('data/'+prefix+'_1_1.npy')
#imx= data[0]
#imy= data[1]
psf_e1 = data[0]
psf_e2 = data[1]
npsf = np.shape(imx)[0]
for ipsf in range(npsf):
#plt.plot(imx[ipsf], imy[ipsf], 'r.')
ang = np.arctan2(psf_e2[ipsf], psf_e1[ipsf])/2
ell = np.sqrt(psf_e1[ipsf]**2 + psf_e2[ipsf]**2)
ell *= 15
lcos = ell*np.cos(ang)
lsin = ell*np.sin(ang)
plt.plot([imx[ipsf]-lcos, imx[ipsf]+lcos],[imy[ipsf]-lsin, imy[ipsf]+lsin],'r', lw=1)
plt.gca().set_aspect(1)
if OVERPLOT == True:
prefix = 'psfEllOPIDW'
plt.savefig('figs/'+prefix+'_iccd{:}.pdf'.format(iccd))
'''
def test_psfdEllPlot():
if ThisTask == 0:
prefix = 'psfEll20'
iccd = 1
iwave= 1
data = np.load('data/'+prefix+'_1_1.npy')
imx= data[0]
imy= data[1]
psf_e1 = data[2]
psf_e2 = data[3]
psf_sz = data[4]
print(np.shape(imx))
npsf = np.shape(imx)[0]
ellX = np.sqrt(psf_e1**2 + psf_e2**2)
angX = np.arctan2(psf_e2, psf_e1)/2
angX = np.rad2deg(angX)
szX = psf_sz
##############################
prefix = 'psfEll20IDW'
data = np.load('data/'+prefix+'_1_1.npy')
#imx= data[0]
#imy= data[1]
psf_e1 = data[0]
psf_e2 = data[1]
psf_sz = data[2]
ellY = np.sqrt(psf_e1**2 + psf_e2**2)
angY = np.arctan2(psf_e2, psf_e1)/2
angY = np.rad2deg(angY)
szY = psf_sz
##############################
fig=plt.figure(figsize=(15, 4))
ax = plt.subplot(1,3,1)
plt.hist(ellX, bins=20, color='b', alpha=0.5)
plt.hist(ellY, bins=20, color='r', alpha=0.5)
plt.xlabel('$\epsilon$')
plt.ylabel('PDF')
ax = plt.subplot(1,3,2)
plt.hist((ellY-ellX)/ellX, bins=20, color='r', alpha=0.5)
plt.xlabel('$(\epsilon_{\\rm IDW}-\epsilon_{\\rm ORG})/\epsilon_{\\rm ORG}$')
plt.ylabel('PDF')
ax = plt.subplot(1,3,3)
plt.hist((angY-angX)/angX, bins=20, color='r', alpha=0.5, range=[-0.1, 0.1])
plt.xlabel('$(\\alpha_{\\rm IDW}-\\alpha_{\\rm ORG})/\\alpha_{\\rm ORG}$')
plt.ylabel('PDF')
plt.savefig('figs/psfEllOPIDWPDF.pdf')
fig=plt.figure(figsize=(4, 4))
plt.hist((szY-szX)/szX, bins=20, color='r', alpha=0.5)
plt.xlabel('$(R_{\\rm IDW}-R_{\\rm ORG})/R_{\\rm ORG}$')
plt.ylabel('PDF')
plt.savefig('figs/psfEllOPIDWPDF_dsz.pdf')
'''
def test_psfdEllabsPlot(iccd):
#if ThisTask == 0:
if True:
prefix = 'psfEll20'
#iccd = 1
#iwave= 1
data = np.load('data/'+prefix+'_{:}_1.npy'.format(iccd))
imx= data[0]
imy= data[1]
psf_e1 = data[2]
psf_e2 = data[3]
psf_sz = data[4]
print(np.shape(imx))
npsf = np.shape(imx)[0]
ellX = np.sqrt(psf_e1**2 + psf_e2**2)
angX = np.arctan2(psf_e2, psf_e1)/2
angX = np.rad2deg(angX)
szX = psf_sz
##############################
prefix = 'psfEll20IDW'
data = np.load('data/'+prefix+'_{:}_1.npy'.format(iccd))
#imx= data[0]
#imy= data[1]
psf_e1 = data[0]
psf_e2 = data[1]
psf_sz = data[2]
ellY = np.sqrt(psf_e1**2 + psf_e2**2)
angY = np.arctan2(psf_e2, psf_e1)/2
angY = np.rad2deg(angY)
szY = psf_sz
##############################
fig=plt.figure(figsize=(6, 5))
grid = plt.GridSpec(3,1,left=0.15, right=0.95, wspace=None, hspace=0.02)
#plt.subplots_adjust(left=None,bottom=None,right=None,top=None,wspace=None,hspace=0.02)
ax = plt.subplot(grid[0:2,0])
plt.plot([0.01,0.1],[0.01,0.1], 'k--', lw=1. )
plt.scatter(ellX, ellY, color='b', alpha=1., s=3., edgecolors='None')
plt.xlim([0.015, 0.085])
plt.ylim([0.015, 0.085])
plt.ylabel('$\epsilon_{\\rm IDW}$')
plt.gca().axes.get_xaxis().set_visible(False)
ax = plt.subplot(grid[2,0])
plt.plot([0.015,0.085],[0.,0.], 'k--', lw=1. )
plt.scatter(ellX, (ellY-ellX), color='b', s=3., edgecolors='None')
plt.xlim([0.015, 0.085])
plt.ylim([-0.0018, 0.0018])
plt.xlabel('$\epsilon_{\\rm ORG}$')
plt.ylabel('$\Delta$')
plt.savefig('figs/psfEllOPIDWPDF_{:}.pdf'.format(iccd))
fig=plt.figure(figsize=(4, 4))
plt.hist((szY-szX)/szX, bins=20, color='r', alpha=0.5)
plt.xlabel('$(R_{\\rm IDW}-R_{\\rm ORG})/R_{\\rm ORG}$')
plt.ylabel('PDF')
plt.savefig('figs/psfEllOPIDWPDF_dsz_{:}.pdf'.format(iccd))
class PSFMatsIDW_coverage(unittest.TestCase):
def test_psfIDW_(self):
#comm = MPI.COMM_WORLD
#ThisTask = comm.Get_rank()
#NTasks = comm.Get_size()
iccd = 1
iwave= 1
ipsf = 400
psfPath = '/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/CSSOS_psf_20210326/CSST_psf_ciomp'
#test_psfIDW(iccd, iwave, psfPath, ThisTask, NTasks)
test_psfIDW(iccd, iwave, psfPath)
'''
for iccd in range(7, 10):
res = np.zeros(400)
for ipsf in range(1,401):
print(ipsf, end="\r")
res[ipsf-1] = test_psfResidualCalc(iccd, iwave, ipsf, psfPath)
#fig = plt.figure(figsize=(6,6))
#plt.hist(np.abs(res), bins=50)
#plt.xlim([0,1])
#plt.savefig('figs/psfResidualREE80PDF.pdf')
print("{:}:".format(iccd), res[res<=0.01].size/400*100)
'''
test_psfResidualPlot(iccd, iwave, ipsf, psfPath)
#test_psfEll(iccd, iwave, ThisTask, NTasks)
test_psfEll(iccd, iwave)
test_psfEllPlot(OVERPLOT=True)
#test_psfdEllPlot()
test_psfdEllabsPlot(iccd)
##############################
##############################
##############################
if __name__=='__main__':
unittest.main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment