From 1635e47d06cd443f859df38625c8a91113bc9705 Mon Sep 17 00:00:00 2001 From: yuedong Date: Sun, 5 Dec 2021 03:47:07 +0800 Subject: [PATCH] modified to absolute import, change location of runExposure.py and run.pbs --- ObservationSim/Instrument/Chip/Chip.py | 18 ++--- ObservationSim/Instrument/Filter.py | 5 +- ObservationSim/MockObject/Catalog.py | 9 +-- ObservationSim/MockObject/CosmicRay.py | 2 +- ObservationSim/MockObject/Galaxy.py | 7 +- ObservationSim/MockObject/MockObject.py | 7 +- ObservationSim/MockObject/Quasar.py | 6 +- ObservationSim/MockObject/SkybackgroundMap.py | 5 +- ObservationSim/MockObject/Star.py | 6 +- ObservationSim/ObservationSim.py | 26 ++++---- ObservationSim/PSF/PSFGauss.py | 4 +- ObservationSim/PSF/PSFInterp/PSFInterp.py | 7 +- ObservationSim/PSF/PSFInterp/PSFProcess.py | 9 +-- ObservationSim/__init__.py | 2 - ObservationSim/_util.py | 12 ---- config/config_sim.yaml | 16 +++-- ObservationSim/run.pbs => run.pbs | 2 +- .../runExposure.py => runExposure.py | 65 +++++++------------ 18 files changed, 86 insertions(+), 122 deletions(-) rename ObservationSim/run.pbs => run.pbs (56%) rename ObservationSim/runExposure.py => runExposure.py (66%) diff --git a/ObservationSim/Instrument/Chip/Chip.py b/ObservationSim/Instrument/Chip/Chip.py index 559fb76..8df5325 100755 --- a/ObservationSim/Instrument/Chip/Chip.py +++ b/ObservationSim/Instrument/Chip/Chip.py @@ -1,30 +1,20 @@ import galsim import os import numpy as np -import Instrument.Chip.Effects as effects -from Instrument.FocalPlane import FocalPlane from astropy.table import Table from numpy.random import Generator, PCG64 -from Config.Header import generatePrimaryHeader, generateExtensionHeader from astropy.io import fits from datetime import datetime +from ObservationSim.Instrument.Chip import Effects as effects +from ObservationSim.Instrument.FocalPlane import FocalPlane +from ObservationSim.Config.Header import generatePrimaryHeader, generateExtensionHeader + class Chip(FocalPlane): def __init__(self, chipID, ccdEffCurve_dir, CRdata_dir, normalize_dir=None, sls_dir=None, config=None, treering_func=None): # Get focal plane (instance of paraent class) info # TODO: use chipID to config individual chip? super().__init__() - # if config is not None: - # self.npix_x = config["npix_x"] - # self.npix_y = config["npix_y"] - # self.read_noise = config["read_noise"] - # self.dark_noise = config["dark_noise"] - # self.pix_scale = config["pix_scale"] - # self.gain = config["gain"] - # self.bias_level = config["bias_level"] - # self.overscan = config["overscan"] - # else: - # Default setting self.npix_x = 9216 self.npix_y = 9232 self.read_noise = 5.0 # e/pix diff --git a/ObservationSim/Instrument/Filter.py b/ObservationSim/Instrument/Filter.py index abad405..082d6a6 100755 --- a/ObservationSim/Instrument/Filter.py +++ b/ObservationSim/Instrument/Filter.py @@ -2,8 +2,9 @@ import galsim import pylab as pl import os import numpy as np -from ._util import photonEnergy -from .FilterParam import FilterParam + +from ObservationSim.Instrument._util import photonEnergy +from ObservationSim.Instrument.FilterParam import FilterParam class Filter(object): def __init__(self, filter_id, filter_type, filter_param, ccd_bandpass): diff --git a/ObservationSim/MockObject/Catalog.py b/ObservationSim/MockObject/Catalog.py index 2ebbcdd..d99f9ce 100644 --- a/ObservationSim/MockObject/Catalog.py +++ b/ObservationSim/MockObject/Catalog.py @@ -4,13 +4,14 @@ import random import galsim import h5py as h5 import healpy as hp -from .Star import Star -from .Galaxy import Galaxy -from .Quasar import Quasar -from ._util import seds, sed_assign, extAv from astropy.table import Table from astropy.coordinates import spherical_to_cartesian +from ObservationSim.MockObject.Star import Star +from ObservationSim.MockObject.Galaxy import Galaxy +from ObservationSim.MockObject.Quasar import Quasar +from ObservationSim.MockObject._util import seds, sed_assign, extAv + NSIDE = 128 class Catalog(object): diff --git a/ObservationSim/MockObject/CosmicRay.py b/ObservationSim/MockObject/CosmicRay.py index 6ebff6f..bdfb9ab 100755 --- a/ObservationSim/MockObject/CosmicRay.py +++ b/ObservationSim/MockObject/CosmicRay.py @@ -1,4 +1,4 @@ -import MockObject +from ObservationSim.MockObject.MockObject import MockObject class CosmicRay(MockObject): pass \ No newline at end of file diff --git a/ObservationSim/MockObject/Galaxy.py b/ObservationSim/MockObject/Galaxy.py index 6d89c15..fdaa80f 100755 --- a/ObservationSim/MockObject/Galaxy.py +++ b/ObservationSim/MockObject/Galaxy.py @@ -3,11 +3,12 @@ import galsim import os, sys import astropy.constants as cons from astropy.table import Table -from ._util import eObs, integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG,convolveGaussXorders -from .SpecDisperser import SpecDisperser -from .MockObject import MockObject from scipy import interpolate +from ObservationSim.MockObject._util import eObs, integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG,convolveGaussXorders +from ObservationSim.MockObject.SpecDisperser import SpecDisperser +from ObservationSim.MockObject.MockObject import MockObject + class Galaxy(MockObject): def __init__(self, param, rotation=None): super().__init__(param) diff --git a/ObservationSim/MockObject/MockObject.py b/ObservationSim/MockObject/MockObject.py index 1cab205..d9fa2b4 100755 --- a/ObservationSim/MockObject/MockObject.py +++ b/ObservationSim/MockObject/MockObject.py @@ -2,9 +2,10 @@ import galsim import numpy as np import astropy.constants as cons from astropy.table import Table -from ._util import magToFlux, vc_A, convolveGaussXorders -from ._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG -from .SpecDisperser import SpecDisperser + +from ObservationSim.MockObject._util import magToFlux, vc_A, convolveGaussXorders +from ObservationSim.MockObject._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG +from ObservationSim.MockObject.SpecDisperser import SpecDisperser class MockObject(object): def __init__(self, param): diff --git a/ObservationSim/MockObject/Quasar.py b/ObservationSim/MockObject/Quasar.py index d88659e..a335de6 100755 --- a/ObservationSim/MockObject/Quasar.py +++ b/ObservationSim/MockObject/Quasar.py @@ -2,11 +2,11 @@ import galsim import os, sys import numpy as np import astropy.constants as cons -from .MockObject import MockObject from astropy.table import Table from scipy import interpolate -from ._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG -from .MockObject import MockObject + +from ObservationSim.MockObject.MockObject import MockObject +from ObservationSim.MockObject._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG class Quasar(MockObject): def __init__(self, param): diff --git a/ObservationSim/MockObject/SkybackgroundMap.py b/ObservationSim/MockObject/SkybackgroundMap.py index 6ae504c..e67f2b9 100644 --- a/ObservationSim/MockObject/SkybackgroundMap.py +++ b/ObservationSim/MockObject/SkybackgroundMap.py @@ -1,6 +1,5 @@ -from .SpecDisperser import SpecDisperser - -from .SpecDisperser import rotate90 +from ObservationSim.MockObject.SpecDisperser import SpecDisperser +from ObservationSim.MockObject.SpecDisperser import rotate90 import galsim import numpy as np diff --git a/ObservationSim/MockObject/Star.py b/ObservationSim/MockObject/Star.py index c387066..b150913 100755 --- a/ObservationSim/MockObject/Star.py +++ b/ObservationSim/MockObject/Star.py @@ -3,11 +3,11 @@ import os, sys import numpy as np import astropy.constants as cons from astropy.table import Table -from ._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG, tag_sed -from .SpecDisperser import SpecDisperser -from .MockObject import MockObject from scipy import interpolate +from ObservationSim.MockObject._util import integrate_sed_bandpass, getNormFactorForSpecWithABMAG, getObservedSED, getABMAG, tag_sed +from ObservationSim.MockObject.MockObject import MockObject + class Star(MockObject): def __init__(self, param): super().__init__(param) diff --git a/ObservationSim/ObservationSim.py b/ObservationSim/ObservationSim.py index 441c540..563feae 100755 --- a/ObservationSim/ObservationSim.py +++ b/ObservationSim/ObservationSim.py @@ -1,15 +1,18 @@ -from Config import ConfigDir, ChipOutput -from Config.Header import generatePrimaryHeader, generateExtensionHeader -from Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip -from MockObject import Catalog, MockObject, Star, Galaxy, Quasar, calculateSkyMap_split_g -from PSF import PSFGauss, FieldDistortion, PSFInterp -from _util import getShearFiled, makeSubDir_PointingList +import sys +import os + +from ObservationSim.Config import ConfigDir, ChipOutput +from ObservationSim.Config.Header import generatePrimaryHeader, generateExtensionHeader +from ObservationSim.Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip +from ObservationSim.MockObject import Catalog, MockObject, Star, Galaxy, Quasar, calculateSkyMap_split_g +from ObservationSim.PSF import PSFGauss, FieldDistortion, PSFInterp +from ObservationSim._util import getShearFiled, makeSubDir_PointingList + from astropy.io import fits from datetime import datetime import numpy as np import mpi4py.MPI as MPI import galsim -import os, sys import logging import psutil @@ -345,10 +348,11 @@ class Observation(object): timestamp_obs = np.tile(timestamp_obs, len(ra_cen)) pointing_type = np.tile(pointing_type, len(ra_cen)) - timestamp_obs = timestamp_obs[pRange] - pointing_type = pointing_type[pRange] - ra_cen = ra_cen[pRange] - dec_cen = dec_cen[pRange] + if pRange is not None: + timestamp_obs = timestamp_obs[pRange] + pointing_type = pointing_type[pRange] + ra_cen = ra_cen[pRange] + dec_cen = dec_cen[pRange] # The Starting pointing ID if pRange is not None: diff --git a/ObservationSim/PSF/PSFGauss.py b/ObservationSim/PSF/PSFGauss.py index 715aa80..5083880 100644 --- a/ObservationSim/PSF/PSFGauss.py +++ b/ObservationSim/PSF/PSFGauss.py @@ -2,8 +2,8 @@ import galsim import sep import numpy as np from scipy.interpolate import interp1d -from .PSFModel import PSFModel -import os, sys + +from ObservationSim.PSF.PSFModel import PSFModel class PSFGauss(PSFModel): def __init__(self, chip, fwhm=0.187, sigSpin=0., psfRa=0.15): diff --git a/ObservationSim/PSF/PSFInterp/PSFInterp.py b/ObservationSim/PSF/PSFInterp/PSFInterp.py index dc3ba25..b3f41c8 100644 --- a/ObservationSim/PSF/PSFInterp/PSFInterp.py +++ b/ObservationSim/PSF/PSFInterp/PSFInterp.py @@ -6,14 +6,13 @@ NOTE: [iccd, iwave, ipsf] are counted from 1 to n, but [tccd, twave, tpsf] are c import galsim import numpy as np - import os import time import copy -from . import PSFConfig as myConfig -from . import PSFUtil as myUtil -from ..PSFModel import PSFModel +from ObservationSim.PSF.PSFInterp import PSFConfig as myConfig +from ObservationSim.PSF.PSFInterp import PSFUtil as myUtil +from ObservationSim.PSF.PSFModel import PSFModel LOG_DEBUG = False #***# NPSF = 900 #***# 30*30 diff --git a/ObservationSim/PSF/PSFInterp/PSFProcess.py b/ObservationSim/PSF/PSFInterp/PSFProcess.py index 39addce..9131ddf 100644 --- a/ObservationSim/PSF/PSFInterp/PSFProcess.py +++ b/ObservationSim/PSF/PSFInterp/PSFProcess.py @@ -3,13 +3,8 @@ import numpy as np import scipy.io import mpi4py.MPI as MPI - - -#sys.path.append("/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_20201222") -# import PSFConfig as myConfig -# import PSFUtil as myUtil -import PSFConfig -import PSFUtil +import ObservationSim.PSF.PSFInterp.PSFConfig as PSFConfig +import ObservationSim.PSF.PSFInterp.PSFUtil as PSFUtil def mkdir(path): isExists = os.path.exists(path) diff --git a/ObservationSim/__init__.py b/ObservationSim/__init__.py index 6d6c240..e69de29 100755 --- a/ObservationSim/__init__.py +++ b/ObservationSim/__init__.py @@ -1,2 +0,0 @@ -import numpy as np -import galsim \ No newline at end of file diff --git a/ObservationSim/_util.py b/ObservationSim/_util.py index 3f839ef..79bda93 100755 --- a/ObservationSim/_util.py +++ b/ObservationSim/_util.py @@ -36,18 +36,6 @@ def imgName(tt=0): return namekey -# def makeSubDir(path_dict, config): -# subImgdir = path_dict["output_img_dir"] + config["mockImgDir"] + '/' -# if not os.path.exists(subImgdir): -# os.system("mkdir %s"%subImgdir) -# imgKey0 = imgName(tt=0) -# imgKey1 = imgName(tt=1) -# imgKey2 = imgName(tt=2) -# subImgdir = subImgdir + imgKey0 + config["reIndex"] + config["reShear"] + "/" -# if not os.path.exists(subImgdir): -# os.system("mkdir %s"%subImgdir) -# return subImgdir, imgKey0, imgKey1, imgKey2 - def makeSubDir_PointingList(path_dict, config, pointing_ID=0): imgDir = os.path.join(path_dict["work_dir"], config["run_name"]) if not os.path.exists(imgDir): diff --git a/config/config_sim.yaml b/config/config_sim.yaml index 6169e7f..595c111 100644 --- a/config/config_sim.yaml +++ b/config/config_sim.yaml @@ -10,11 +10,15 @@ # Can add some of the command-line arguments here as well; # OK to pass either way or both, as long as they are consistent # work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/" -work_dir: "/public/home/fangyuedong/test_release/CSST/workplace/" +work_dir: "/public/home/fangyuedong/20211203/CSST/workplace/" data_dir: "/data/simudata/CSSOSDataProductsSims/data/" run_name: "TEST" # (Optional) a file of point list +# if you just want to run default pointing: +# - pointing_dir: null +# - pointing_file: null +pointing_dir: "/data/simudata/CSSOSDataProductsSims/data/" pointing_file: "pointing10_20210202.dat" # Whether to use MPI @@ -63,13 +67,17 @@ obs_setting: # Note: only valid when a pointing list is specified np_cal: 0 - # (Optional) only run specific pointing(s). + # Run specific pointing(s): + # - give a list of indexes of pointings: [ip_1, ip_2...] + # - run all pointings: null # Note: only valid when a pointing list is specified run_pointings: [5, 6] - # (Optional) only run specific chip(s) + # Run specific chip(s): + # - give a list of indexes of chips: [ip_1, ip_2...] + # - run all chips: null # Note: for all pointings - # run_chips: [6, 25] + run_chips: [1, 25] ############################################### # Input path setting diff --git a/ObservationSim/run.pbs b/run.pbs similarity index 56% rename from ObservationSim/run.pbs rename to run.pbs index 06bc354..497e0c6 100755 --- a/ObservationSim/run.pbs +++ b/run.pbs @@ -16,4 +16,4 @@ NP=40 date echo $NP -mpirun -np $NP python /public/home/fangyuedong/test_release/CSST/ObservationSim/runExposure.py config_sim.yaml -c /public/home/fangyuedong/test_release/CSST/config +mpirun -np $NP python /public/home/fangyuedong/20211203/CSST/runExposure.py config_sim.yaml -c /public/home/fangyuedong/20211203/CSST/config diff --git a/ObservationSim/runExposure.py b/runExposure.py similarity index 66% rename from ObservationSim/runExposure.py rename to runExposure.py index 8ae357c..c77e1cd 100755 --- a/ObservationSim/runExposure.py +++ b/runExposure.py @@ -1,5 +1,5 @@ -from ObservationSim import Observation -from _util import parse_args +from ObservationSim.ObservationSim import Observation +from ObservationSim._util import parse_args from datetime import datetime import os import numpy as np @@ -9,20 +9,24 @@ import yaml import gc gc.enable() -def Pointing(config, pointing_filename, data_dir): - pointing_file = os.path.join(data_dir, pointing_filename) - f = open(pointing_file, 'r') - for _ in range(1): - header = f.readline() - iline = 0 +def Pointing(config, pointing_filename=None, data_dir=None): pRA = [] pDEC = [] - for line in f: - line = line.strip() - columns = line.split() - pRA.append(float(columns[0])) - pDEC.append(float(columns[1])) - f.close() + if pointing_filename and data_dir: + pointing_file = os.path.join(data_dir, pointing_filename) + f = open(pointing_file, 'r') + for _ in range(1): + header = f.readline() + iline = 0 + for line in f: + line = line.strip() + columns = line.split() + pRA.append(float(columns[0])) + pDEC.append(float(columns[1])) + f.close() + else: + pRa.append(config["obs_setting"]["ra_center"]) + pDec.append(config["obs_setting"]["dec_center"]) pRA = np.array(pRA) pDEC = np.array(pDEC) @@ -43,7 +47,7 @@ def Pointing(config, pointing_filename, data_dir): for i in range(len(pointing_type)): timestamp_obs.append(t) if pointing_type[i] == 'CAL': - t += 3 * delta_t * 60 # 3 calibration exposure + t += 3 * delta_t * 60 # 3 calibration exposures for each pointing elif pointing_type[i] == 'MS': t += delta_t * 60 timestamp_obs = np.array(timestamp_obs) @@ -98,22 +102,13 @@ def runSim(): if args.work_dir is not None: config['work_dir'] = args.work_dir - pRA, pDEC, timestamp_obs, pointing_type = Pointing(config=config, pointing_filename=config['pointing_file'], data_dir=config['data_dir']) + pRA, pDEC, timestamp_obs, pointing_type = Pointing(config=config, pointing_filename=config['pointing_file'], data_dir=config['pointing_dir']) MakeDirectories(work_dir=config['work_dir'], run_name=config['run_name'], nPointings=len(pRA), pRange=config['obs_setting']['run_pointings']) - if "run_chips" in config["obs_setting"]: - run_chips = config["obs_setting"]["run_chips"] - else: - run_chips = None - - # from Config import ConfigDir - # path_dict = ConfigDir(config=config, work_dir=config["work_dir"], data_dir=config["data_dir"]) - # for key, value in path_dict.items(): - # print (key + " : " + str(value)) obs = Observation(config=config, work_dir=config['work_dir'], data_dir=config['data_dir']) if config["pointing_file"] is None: - obs.runExposure(chips=run_chips) + obs.runExposure(chips=config["obs_setting"]["run_chips"]) else: obs.runExposure_MPI_PointingList( ra_cen=pRA, @@ -123,24 +118,8 @@ def runSim(): pointing_type=pointing_type, exptime=config["obs_setting"]["exp_time"], use_mpi=config["run_option"]["use_mpi"], - chips=run_chips + chips=config["obs_setting"]["run_chips"] ) - print("run finished") if __name__=='__main__': runSim() - -############################################# -# Testing run one exposure (NOT using MPI) -# ipoint = 2 -# pointRa = float('{:8.4f}'.format(pRA[ipoint])) -# pointDec = float('{:8.4f}'.format(pDEC[ipoint])) - -# obs = Observation(work_dir=work_dir, data_dir=data_dir) -# obs.runExposure(ra_cen=pointRa, dec_cen=pointDec, pointing_ID=ipoint, oneChip=16) - -############################################# -# Testing run pointing list (using MPI) -# obs = Observation(work_dir=work_dir, data_dir=data_dir) -# # obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type) -# obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange, timestamp_obs=timestamp_obs, pointing_type=pointing_type, exptime=exp_time) -- GitLab