diff --git a/ObservationSim/Instrument/Chip/Chip.py b/ObservationSim/Instrument/Chip/Chip.py index 559fb7663c5815879fc9df0d703f17f430694ab5..8df53258444847dd8483a21d58ecbe54d47eca76 100755 --- a/ObservationSim/Instrument/Chip/Chip.py +++ b/ObservationSim/Instrument/Chip/Chip.py @@ -1,30 +1,20 @@ import galsim import os import numpy as np -import Instrument.Chip.Effects as effects -from Instrument.FocalPlane import FocalPlane from astropy.table import Table from numpy.random import Generator, PCG64 -from Config.Header import generatePrimaryHeader, generateExtensionHeader from astropy.io import fits from datetime import datetime +from ObservationSim.Instrument.Chip import Effects as effects +from ObservationSim.Instrument.FocalPlane import FocalPlane +from ObservationSim.Config.Header import generatePrimaryHeader, generateExtensionHeader + class Chip(FocalPlane): def __init__(self, chipID, ccdEffCurve_dir, CRdata_dir, normalize_dir=None, sls_dir=None, config=None, treering_func=None): # Get focal plane (instance of paraent class) info # TODO: use chipID to config individual chip? super().__init__() - # if config is not None: - # self.npix_x = config["npix_x"] - # self.npix_y = config["npix_y"] - # self.read_noise = config["read_noise"] - # self.dark_noise = config["dark_noise"] - # self.pix_scale = config["pix_scale"] - # self.gain = config["gain"] - # self.bias_level = config["bias_level"] - # self.overscan = config["overscan"] - # else: - # Default setting self.npix_x = 9216 self.npix_y = 9232 self.read_noise = 5.0 # e/pix diff --git a/ObservationSim/Instrument/Filter.py b/ObservationSim/Instrument/Filter.py index abad4059c3a902a68f0fb700f20f1d60ce5d2a7f..082d6a6e75cc37f21837ba741f6ddd8248f6f7ba 100755 --- a/ObservationSim/Instrument/Filter.py +++ b/ObservationSim/Instrument/Filter.py @@ -2,8 +2,9 @@ import galsim import pylab as pl import os import numpy as np -from ._util import photonEnergy -from .FilterParam import FilterParam + +from ObservationSim.Instrument._util import photonEnergy +from ObservationSim.Instrument.FilterParam import FilterParam class Filter(object): def __init__(self, filter_id, filter_type, filter_param, ccd_bandpass): diff --git a/ObservationSim/MockObject/Catalog.py b/ObservationSim/MockObject/Catalog.py index 2ebbcdd697d759150566f054e25f243dd3ccc64c..d99f9cec29e76dcd1f429c12a0668e857a2dc404 100644 --- a/ObservationSim/MockObject/Catalog.py +++ b/ObservationSim/MockObject/Catalog.py @@ -4,13 +4,14 @@ import random import galsim import h5py as h5 import healpy as hp -from .Star import Star -from .Galaxy import Galaxy -from .Quasar import Quasar -from ._util import seds, sed_assign, extAv from astropy.table import Table from astropy.coordinates import spherical_to_cartesian +from ObservationSim.MockObject.Star import Star +from ObservationSim.MockObject.Galaxy import Galaxy +from ObservationSim.MockObject.Quasar import Quasar +from ObservationSim.MockObject._util import seds, sed_assign, extAv + NSIDE = 128 class Catalog(object): diff --git a/ObservationSim/MockObject/CosmicRay.py b/ObservationSim/MockObject/CosmicRay.py index 6ebff6f398bfa0f80a974423ea63bb9a65f4f161..bdfb9ab3ae1fe2705b42b347592bde7c11649a6c 100755 --- a/ObservationSim/MockObject/CosmicRay.py +++ b/ObservationSim/MockObject/CosmicRay.py @@ -1,4 +1,4 @@ -import MockObject +from ObservationSim.MockObject.MockObject import MockObject class CosmicRay(MockObject): pass \ No newline at end of file diff --git a/ObservationSim/MockObject/Galaxy.py b/ObservationSim/MockObject/Galaxy.py index 6d89c15efc64c564dce3967bea349d412b1f87a9..fdaa80f91a33d1e1b97f43fd6859a001dc273adf 100755 --- a/ObservationSim/MockObject/Galaxy.py +++ b/ObservationSim/MockObject/Galaxy.py @@ -3,11 +3,12 @@ import galsim import os, sys import astropy.constants as cons from astropy.table import Table -from ._util import eObs, integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG,convolveGaussXorders -from .SpecDisperser import SpecDisperser -from .MockObject import MockObject from scipy import interpolate +from ObservationSim.MockObject._util import eObs, integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG,convolveGaussXorders +from ObservationSim.MockObject.SpecDisperser import SpecDisperser +from ObservationSim.MockObject.MockObject import MockObject + class Galaxy(MockObject): def __init__(self, param, rotation=None): super().__init__(param) diff --git a/ObservationSim/MockObject/MockObject.py b/ObservationSim/MockObject/MockObject.py index 1cab2054eaed86e43f0a39e71bf04df1bb58d1c7..d9fa2b43cf2e00c29ed76c1266c7d1ca4a932be4 100755 --- a/ObservationSim/MockObject/MockObject.py +++ b/ObservationSim/MockObject/MockObject.py @@ -2,9 +2,10 @@ import galsim import numpy as np import astropy.constants as cons from astropy.table import Table -from ._util import magToFlux, vc_A, convolveGaussXorders -from ._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG -from .SpecDisperser import SpecDisperser + +from ObservationSim.MockObject._util import magToFlux, vc_A, convolveGaussXorders +from ObservationSim.MockObject._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG +from ObservationSim.MockObject.SpecDisperser import SpecDisperser class MockObject(object): def __init__(self, param): diff --git a/ObservationSim/MockObject/Quasar.py b/ObservationSim/MockObject/Quasar.py index d88659e064e371aa5cddca36818e375d9025259a..a335de6ae92ad12e8c27004b1bd5f81636b66fff 100755 --- a/ObservationSim/MockObject/Quasar.py +++ b/ObservationSim/MockObject/Quasar.py @@ -2,11 +2,11 @@ import galsim import os, sys import numpy as np import astropy.constants as cons -from .MockObject import MockObject from astropy.table import Table from scipy import interpolate -from ._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG -from .MockObject import MockObject + +from ObservationSim.MockObject.MockObject import MockObject +from ObservationSim.MockObject._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG class Quasar(MockObject): def __init__(self, param): diff --git a/ObservationSim/MockObject/SkybackgroundMap.py b/ObservationSim/MockObject/SkybackgroundMap.py index 6ae504ce81b8f22796a0d2c60f9db410fb1d15cc..e67f2b9a300efd699d8a37845e9c7dc1d49ac8ab 100644 --- a/ObservationSim/MockObject/SkybackgroundMap.py +++ b/ObservationSim/MockObject/SkybackgroundMap.py @@ -1,6 +1,5 @@ -from .SpecDisperser import SpecDisperser - -from .SpecDisperser import rotate90 +from ObservationSim.MockObject.SpecDisperser import SpecDisperser +from ObservationSim.MockObject.SpecDisperser import rotate90 import galsim import numpy as np diff --git a/ObservationSim/MockObject/Star.py b/ObservationSim/MockObject/Star.py index c3870665a8f14a8f30282cba37caa6146ace6efd..b1509130d0145582644a6569cfa43d35982cd235 100755 --- a/ObservationSim/MockObject/Star.py +++ b/ObservationSim/MockObject/Star.py @@ -3,11 +3,11 @@ import os, sys import numpy as np import astropy.constants as cons from astropy.table import Table -from ._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG, tag_sed -from .SpecDisperser import SpecDisperser -from .MockObject import MockObject from scipy import interpolate +from ObservationSim.MockObject._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG, tag_sed +from ObservationSim.MockObject.MockObject import MockObject + class Star(MockObject): def __init__(self, param): super().__init__(param) diff --git a/ObservationSim/ObservationSim.py b/ObservationSim/ObservationSim.py index 441c540eb181ff572728edcc7eb840556ba61ba7..563feaeee3c348efded5e3322d748b7be741060d 100755 --- a/ObservationSim/ObservationSim.py +++ b/ObservationSim/ObservationSim.py @@ -1,15 +1,18 @@ -from Config import ConfigDir, ChipOutput -from Config.Header import generatePrimaryHeader, generateExtensionHeader -from Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip -from MockObject import Catalog, MockObject, Star, Galaxy, Quasar, calculateSkyMap_split_g -from PSF import PSFGauss, FieldDistortion, PSFInterp -from _util import getShearFiled, makeSubDir_PointingList +import sys +import os + +from ObservationSim.Config import ConfigDir, ChipOutput +from ObservationSim.Config.Header import generatePrimaryHeader, generateExtensionHeader +from ObservationSim.Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip +from ObservationSim.MockObject import Catalog, MockObject, Star, Galaxy, Quasar, calculateSkyMap_split_g +from ObservationSim.PSF import PSFGauss, FieldDistortion, PSFInterp +from ObservationSim._util import getShearFiled, makeSubDir_PointingList + from astropy.io import fits from datetime import datetime import numpy as np import mpi4py.MPI as MPI import galsim -import os, sys import logging import psutil @@ -345,10 +348,11 @@ class Observation(object): timestamp_obs = np.tile(timestamp_obs, len(ra_cen)) pointing_type = np.tile(pointing_type, len(ra_cen)) - timestamp_obs = timestamp_obs[pRange] - pointing_type = pointing_type[pRange] - ra_cen = ra_cen[pRange] - dec_cen = dec_cen[pRange] + if pRange is not None: + timestamp_obs = timestamp_obs[pRange] + pointing_type = pointing_type[pRange] + ra_cen = ra_cen[pRange] + dec_cen = dec_cen[pRange] # The Starting pointing ID if pRange is not None: diff --git a/ObservationSim/PSF/PSFGauss.py b/ObservationSim/PSF/PSFGauss.py index 715aa80326dcf6ae466b8eac41066645b20ac0f0..508388055bec2e652418f5d41e69685b2d9c0191 100644 --- a/ObservationSim/PSF/PSFGauss.py +++ b/ObservationSim/PSF/PSFGauss.py @@ -2,8 +2,8 @@ import galsim import sep import numpy as np from scipy.interpolate import interp1d -from .PSFModel import PSFModel -import os, sys + +from ObservationSim.PSF.PSFModel import PSFModel class PSFGauss(PSFModel): def __init__(self, chip, fwhm=0.187, sigSpin=0., psfRa=0.15): diff --git a/ObservationSim/PSF/PSFInterp/PSFInterp.py b/ObservationSim/PSF/PSFInterp/PSFInterp.py index dc3ba25916506cb02bef4dffa1a8d498475b1033..b3f41c8a87f266de032433b71937809dd0e0cc69 100644 --- a/ObservationSim/PSF/PSFInterp/PSFInterp.py +++ b/ObservationSim/PSF/PSFInterp/PSFInterp.py @@ -6,14 +6,13 @@ NOTE: [iccd, iwave, ipsf] are counted from 1 to n, but [tccd, twave, tpsf] are c import galsim import numpy as np - import os import time import copy -from . import PSFConfig as myConfig -from . import PSFUtil as myUtil -from ..PSFModel import PSFModel +from ObservationSim.PSF.PSFInterp import PSFConfig as myConfig +from ObservationSim.PSF.PSFInterp import PSFUtil as myUtil +from ObservationSim.PSF.PSFModel import PSFModel LOG_DEBUG = False #***# NPSF = 900 #***# 30*30 diff --git a/ObservationSim/PSF/PSFInterp/PSFProcess.py b/ObservationSim/PSF/PSFInterp/PSFProcess.py index 39addcea688d3e2554cf482b021aa3ac64d1cf18..9131ddfd8a206be5de9904e1f508bfb87799186d 100644 --- a/ObservationSim/PSF/PSFInterp/PSFProcess.py +++ b/ObservationSim/PSF/PSFInterp/PSFProcess.py @@ -3,13 +3,8 @@ import numpy as np import scipy.io import mpi4py.MPI as MPI - - -#sys.path.append("/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_20201222") -# import PSFConfig as myConfig -# import PSFUtil as myUtil -import PSFConfig -import PSFUtil +import ObservationSim.PSF.PSFInterp.PSFConfig as PSFConfig +import ObservationSim.PSF.PSFInterp.PSFUtil as PSFUtil def mkdir(path): isExists = os.path.exists(path) diff --git a/ObservationSim/__init__.py b/ObservationSim/__init__.py index 6d6c2403494d15afc8d835480ec5f47f7ee5217a..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100755 --- a/ObservationSim/__init__.py +++ b/ObservationSim/__init__.py @@ -1,2 +0,0 @@ -import numpy as np -import galsim \ No newline at end of file diff --git a/ObservationSim/_util.py b/ObservationSim/_util.py index 3f839ef29f92096e5a3b8b4df448360068f00d4f..79bda9344db54331b41e01d0ee9e59ef74e40248 100755 --- a/ObservationSim/_util.py +++ b/ObservationSim/_util.py @@ -36,18 +36,6 @@ def imgName(tt=0): return namekey -# def makeSubDir(path_dict, config): -# subImgdir = path_dict["output_img_dir"] + config["mockImgDir"] + '/' -# if not os.path.exists(subImgdir): -# os.system("mkdir %s"%subImgdir) -# imgKey0 = imgName(tt=0) -# imgKey1 = imgName(tt=1) -# imgKey2 = imgName(tt=2) -# subImgdir = subImgdir + imgKey0 + config["reIndex"] + config["reShear"] + "/" -# if not os.path.exists(subImgdir): -# os.system("mkdir %s"%subImgdir) -# return subImgdir, imgKey0, imgKey1, imgKey2 - def makeSubDir_PointingList(path_dict, config, pointing_ID=0): imgDir = os.path.join(path_dict["work_dir"], config["run_name"]) if not os.path.exists(imgDir): diff --git a/config/config_sim.yaml b/config/config_sim.yaml index 6169e7f3028f4febfceaf6f493a5b2dd40eaa996..595c11133bb6f12a45639e2b850a53a06b6e6354 100644 --- a/config/config_sim.yaml +++ b/config/config_sim.yaml @@ -10,11 +10,15 @@ # Can add some of the command-line arguments here as well; # OK to pass either way or both, as long as they are consistent # work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/" -work_dir: "/public/home/fangyuedong/test_release/CSST/workplace/" +work_dir: "/public/home/fangyuedong/20211203/CSST/workplace/" data_dir: "/data/simudata/CSSOSDataProductsSims/data/" run_name: "TEST" # (Optional) a file of point list +# if you just want to run default pointing: +# - pointing_dir: null +# - pointing_file: null +pointing_dir: "/data/simudata/CSSOSDataProductsSims/data/" pointing_file: "pointing10_20210202.dat" # Whether to use MPI @@ -63,13 +67,17 @@ obs_setting: # Note: only valid when a pointing list is specified np_cal: 0 - # (Optional) only run specific pointing(s). + # Run specific pointing(s): + # - give a list of indexes of pointings: [ip_1, ip_2...] + # - run all pointings: null # Note: only valid when a pointing list is specified run_pointings: [5, 6] - # (Optional) only run specific chip(s) + # Run specific chip(s): + # - give a list of indexes of chips: [ip_1, ip_2...] + # - run all chips: null # Note: for all pointings - # run_chips: [6, 25] + run_chips: [1, 25] ############################################### # Input path setting diff --git a/ObservationSim/run.pbs b/run.pbs similarity index 56% rename from ObservationSim/run.pbs rename to run.pbs index 06bc354680ecce9821adecce5fe0b8c89aa4a60b..497e0c6a07c9c321d418161c8b2374b53a0bae8c 100755 --- a/ObservationSim/run.pbs +++ b/run.pbs @@ -16,4 +16,4 @@ NP=40 date echo $NP -mpirun -np $NP python /public/home/fangyuedong/test_release/CSST/ObservationSim/runExposure.py config_sim.yaml -c /public/home/fangyuedong/test_release/CSST/config +mpirun -np $NP python /public/home/fangyuedong/20211203/CSST/runExposure.py config_sim.yaml -c /public/home/fangyuedong/20211203/CSST/config diff --git a/ObservationSim/runExposure.py b/runExposure.py similarity index 66% rename from ObservationSim/runExposure.py rename to runExposure.py index 8ae357c8f396ee7de17c221aea2ea0a2909e0e91..c77e1cd59efbdc8681154fe4ab2b5a8d4a970482 100755 --- a/ObservationSim/runExposure.py +++ b/runExposure.py @@ -1,5 +1,5 @@ -from ObservationSim import Observation -from _util import parse_args +from ObservationSim.ObservationSim import Observation +from ObservationSim._util import parse_args from datetime import datetime import os import numpy as np @@ -9,20 +9,24 @@ import yaml import gc gc.enable() -def Pointing(config, pointing_filename, data_dir): - pointing_file = os.path.join(data_dir, pointing_filename) - f = open(pointing_file, 'r') - for _ in range(1): - header = f.readline() - iline = 0 +def Pointing(config, pointing_filename=None, data_dir=None): pRA = [] pDEC = [] - for line in f: - line = line.strip() - columns = line.split() - pRA.append(float(columns[0])) - pDEC.append(float(columns[1])) - f.close() + if pointing_filename and data_dir: + pointing_file = os.path.join(data_dir, pointing_filename) + f = open(pointing_file, 'r') + for _ in range(1): + header = f.readline() + iline = 0 + for line in f: + line = line.strip() + columns = line.split() + pRA.append(float(columns[0])) + pDEC.append(float(columns[1])) + f.close() + else: + pRa.append(config["obs_setting"]["ra_center"]) + pDec.append(config["obs_setting"]["dec_center"]) pRA = np.array(pRA) pDEC = np.array(pDEC) @@ -43,7 +47,7 @@ def Pointing(config, pointing_filename, data_dir): for i in range(len(pointing_type)): timestamp_obs.append(t) if pointing_type[i] == 'CAL': - t += 3 * delta_t * 60 # 3 calibration exposure + t += 3 * delta_t * 60 # 3 calibration exposures for each pointing elif pointing_type[i] == 'MS': t += delta_t * 60 timestamp_obs = np.array(timestamp_obs) @@ -98,22 +102,13 @@ def runSim(): if args.work_dir is not None: config['work_dir'] = args.work_dir - pRA, pDEC, timestamp_obs, pointing_type = Pointing(config=config, pointing_filename=config['pointing_file'], data_dir=config['data_dir']) + pRA, pDEC, timestamp_obs, pointing_type = Pointing(config=config, pointing_filename=config['pointing_file'], data_dir=config['pointing_dir']) MakeDirectories(work_dir=config['work_dir'], run_name=config['run_name'], nPointings=len(pRA), pRange=config['obs_setting']['run_pointings']) - if "run_chips" in config["obs_setting"]: - run_chips = config["obs_setting"]["run_chips"] - else: - run_chips = None - - # from Config import ConfigDir - # path_dict = ConfigDir(config=config, work_dir=config["work_dir"], data_dir=config["data_dir"]) - # for key, value in path_dict.items(): - # print (key + " : " + str(value)) obs = Observation(config=config, work_dir=config['work_dir'], data_dir=config['data_dir']) if config["pointing_file"] is None: - obs.runExposure(chips=run_chips) + obs.runExposure(chips=config["obs_setting"]["run_chips"]) else: obs.runExposure_MPI_PointingList( ra_cen=pRA, @@ -123,24 +118,8 @@ def runSim(): pointing_type=pointing_type, exptime=config["obs_setting"]["exp_time"], use_mpi=config["run_option"]["use_mpi"], - chips=run_chips + chips=config["obs_setting"]["run_chips"] ) - print("run finished") if __name__=='__main__': runSim() - -############################################# -# Testing run one exposure (NOT using MPI) -# ipoint = 2 -# pointRa = float('{:8.4f}'.format(pRA[ipoint])) -# pointDec = float('{:8.4f}'.format(pDEC[ipoint])) - -# obs = Observation(work_dir=work_dir, data_dir=data_dir) -# obs.runExposure(ra_cen=pointRa, dec_cen=pointDec, pointing_ID=ipoint, oneChip=16) - -############################################# -# Testing run pointing list (using MPI) -# obs = Observation(work_dir=work_dir, data_dir=data_dir) -# # obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type) -# obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=exp_time)