Skip to content
GitLab
Commit 3a9baf79 authored by Fang Yuedong's avatar Fang Yuedong
Browse files

Merge branch 'sim_scheduler' into 'develop' 

package name and version change

See merge request csst-sims/csst_msc_sim!19
parents af8ceb6c 23ec8465
Pipeline #4236 passed with stage
in 0 seconds
Hide whitespace changes
Inline Side-by-side
.gitignore
View file @ 3a9baf79
build/*
CSSTSim.egg-info/*
csst_msc_sim.egg-info/*
dist/*
*.pyc
*.so
......
Catalog/Catalog_example.py
View file @ 3a9baf79
......@@ -53,7 +53,7 @@ class Catalog(CatalogBase):
super().__init__()
self.cat_dir = os.path.join(config["data_dir"], config["catalog_options"]["input_path"]["cat_dir"])
self.chip = chip
if "star_cat" in config["catalog_options"]["input_path"] and config["catalog_options"]["input_path"]["star_cat"]
if "star_cat" in config["catalog_options"]["input_path"] and config["catalog_options"]["input_path"]["star_cat"]:
star_file = config["catalog_options"]["input_path"]["star_cat"]
star_SED_file = config["catalog_options"]["SED_templates_path"]["star_SED"]
self.star_path = os.path.join(self.cat_dir, star_file)
......
ObservationSim/Config/Header/ImageHeader.py
View file @ 3a9baf79
......@@ -473,7 +473,7 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointing_id = '00000001', po
# h_prim['SIM_VER'] = (get_distribution("CSSTSim").version, "Version of CSST MSC simulation software")
currentDateAndTime = datetime.now()
compute_name = platform.node()
h_prim['FITSSWV'] = get_distribution("CSSTSim").version +'_' + currentDateAndTime.strftime("%Y%m%d") + '_' +compute_name
h_prim['FITSSWV'] = get_distribution("csst_msc_sim").version +'_' + currentDateAndTime.strftime("%Y%m%d") + '_' +compute_name
h_prim['EPOCH'] = round((Time(h_prim['EXPSTART'], format='mjd', scale='tcb')).jyear, 1)
return h_prim
......
ObservationSim/MockObject/FlatLED.py
View file @ 3a9baf79
......@@ -52,7 +52,7 @@ fluxLED = {'LED1': 15, 'LED2': 15, 'LED3': 12.5, 'LED4': 9, 'LED5': 9,
mirro_eff = {'GU':0.61, 'GV':0.8, 'GI':0.8}
# mirro_eff = {'GU':1, 'GV':1, 'GI':1}
class FlatLED(object):
class FlatLED(MockObject):
def __init__(self, chip,filt, flatDir = None, logger=None):
# self.led_type_list = led_type_list
self.filt = filt
......@@ -426,7 +426,7 @@ class FlatLED(object):
print('time: %s ms' % ((T2 - T1) * 1000))
if isAlongY == 1:
fimg, tmx, tmy = rotate90(array_orig=fImg.array, xc=0, yc=0, isClockwise=0)
fimg, tmx, tmy = SpecDisperser.rotate90(array_orig=fImg.array, xc=0, yc=0, isClockwise=0)
else:
fimg = fImg.array
......
ObservationSim/sim_steps/add_LED_flat.py
View file @ 3a9baf79
......@@ -29,7 +29,7 @@ def add_LED_Flat(self, chip, filt, tel, pointing, catalog, obs_param):
pf_map = np.array(pf_map, dtype='float32')
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True])
else:
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [True,'','','',''])
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN1','SHTCLOS0'], values = [True,self.h_ext['SHTCLOS1'],self.h_ext['SHTOPEN0']])
chip.img = chip.img + pf_map
......
ObservationSim/sim_steps/add_objects.py
View file @ 3a9baf79
......@@ -211,7 +211,7 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
flat_normal = np.array(flat_normal, dtype='float32')
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True])
else:
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [True,'','','',''])
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN1','SHTCLOS0'], values = [True,self.h_ext['SHTCLOS1'],self.h_ext['SHTOPEN0']])
chip.img *= flat_normal
del flat_normal
......
ObservationSim/sim_steps/add_sky_background.py
View file @ 3a9baf79
......@@ -22,7 +22,7 @@ def add_sky_background_sci(self, chip, filt, tel, pointing, catalog, obs_param):
flat_normal = np.array(flat_normal, dtype='float32')
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True])
else:
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [False,'','','',''])
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN1','SHTCLOS0'], values = [True,self.h_ext['SHTCLOS1'],self.h_ext['SHTOPEN0']])
if obs_param["enable_straylight_model"]:
# Filter.sky_background, Filter.zodical_spec will be updated
......@@ -84,7 +84,7 @@ def add_sky_flat_calibration(self, chip, filt, tel, pointing, catalog, obs_param
shutt_gsimg.write("%s/ShutterEffect_%s_1.fits" % (self.chip_output.subdir, str(chip.chipID).rjust(2, '0')))
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True])
else:
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [True,'','','',''])
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN1','SHTCLOS0'], values = [True,self.h_ext['SHTCLOS1'],self.h_ext['SHTOPEN0']])
if chip.survey_type == "photometric":
......
ObservationSim/sim_steps/readout_output.py
View file @ 3a9baf79
......@@ -45,7 +45,7 @@ def quantization_and_output(self, chip, filt, tel, pointing, catalog, obs_param)
if not hasattr(self, 'h_ext'):
_, _ = self.prepare_headers(chip=chip, pointing=pointing)
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1','EXPTIME'], values = [False,'','','','',0.0])
self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN1','SHTCLOS0','SHTCLOS1','EXPTIME'], values = [False,self.h_ext['SHTOPEN0'],self.h_ext['SHTOPEN0'],self.h_ext['SHTOPEN0'],0.0])
# renew header info
datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
datetime_obs = datetime_obs.replace(tzinfo=timezone.utc)
......
config/config_overall.yaml
View file @ 3a9baf79
......@@ -46,6 +46,9 @@ catalog_options:
# Only simulate galaxies?
galaxy_only: YES
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
###############################################
# Observation setting
###############################################
......@@ -56,7 +59,7 @@ obs_setting:
# - pointing_file: null
pointing_file: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/pointing50_C9/pointing_50_1_n.dat"
obs_config_file: "/public/home/fangyuedong/project/csst-simulation/config/obs_config_SCI_WIDE_phot.yaml"
obs_config_file: "/public/home/fangyuedong/project/csst_msc_sim/config/obs_config_SCI_WIDE_phot.yaml"
# Run specific pointing(s):
# - give a list of indexes of pointings: [ip_1, ip_2...]
......
config/config_overall_newStar.yaml
View file @ 3a9baf79
......@@ -46,6 +46,9 @@ catalog_options:
# Only simulate galaxies?
galaxy_only: NO
# rotate galaxy ellipticity
rotateEll: 0. # [degree]
###############################################
# Observation setting
###############################################
......
install.sh
View file @ 3a9baf79
......@@ -5,5 +5,5 @@ conda install numpy==1.26.4
conda install cython==3.0.6
conda install sep==1.2.1
conda install mpi4py==3.1.6
python3 setup.py install --user
conda install cfitsio==3.470
pip install -e .
run_sim.py
View file @ 3a9baf79
......@@ -10,6 +10,7 @@ import importlib
import gc
gc.enable()
def run_sim():
"""
Main method for simulation call.
......@@ -18,13 +19,13 @@ def run_sim():
----------
Catalog : Class
a catalog class which is inherited from ObservationSim.MockObject.CatalogBase
Returns
----------
None
"""
# Get version of CSSTSim Package
__version__ = version("CSSTSim")
# Get version of the Package
__version__ = version("csst_msc_sim")
# Get run datetime
now = datetime.datetime.now()
......@@ -38,7 +39,7 @@ def run_sim():
try:
config = yaml.safe_load(stream)
for key, value in config.items():
print (key + " : " + str(value))
print(key + " : " + str(value))
except yaml.YAMLError as exc:
print(exc)
......@@ -62,42 +63,48 @@ def run_sim():
if "run_counter" not in config:
config["run_counter"] = 0
# Generate lists pointings based on the input pointing list (or default
# Generate lists pointings based on the input pointing list (or default
# pointing RA, DEC) and "config["obs_setting"]["run_pointings"]".
# "config['obs_setting']['np_cal']"" is the number of CAL pointings which will be
# "config['obs_setting']['np_cal']"" is the number of CAL pointings which will be
# appended to the front.
# NOTE: the implementation of gerenating time_stamps is temporary.
pointing_dir = None
if "pointing_dir" in config['obs_setting']:
pointing_dir = config['obs_setting']["pointing_dir"]
pointing_list = generate_pointing_list(config=config, pointing_filename=config['obs_setting']['pointing_file'], data_dir=pointing_dir)
pointing_list = generate_pointing_list(
config=config, pointing_filename=config['obs_setting']['pointing_file'], data_dir=pointing_dir)
# Make the main output directories
run_dir = make_run_dirs(work_dir=config['work_dir'], run_name=config['run_name'], pointing_list=pointing_list)
run_dir = make_run_dirs(
work_dir=config['work_dir'], run_name=config['run_name'], pointing_list=pointing_list)
# Copy the config file to output directory & Write Run metadata
shutil.copy(args.config_file, run_dir)
run_meta = os.path.join(run_dir, "run_metadata.yaml")
with open(run_meta, "w") as config_out:
config_out.write("\n")
config_out.write("###############################################\n")
config_out.write("CSSTSim_version: \"%s\"\n"%__version__)
config_out.write(
"csst_msc_sim package version: \"%s\"\n" % __version__)
date_str = datetime.datetime.strftime(now, '%m/%d/%Y')
time_str = datetime.datetime.strftime(now, '%H:%M:%S')
config_out.write("Run_date: \"%s\"\n"%date_str)
config_out.write("Run_time: \"%s\"\n"%time_str)
config_out.write("Run_date: \"%s\"\n" % date_str)
config_out.write("Run_time: \"%s\"\n" % time_str)
config_out.write("###############################################\n")
# Initialize the simulation
if args.catalog is not None:
catalog_module = importlib.import_module('Catalog.'+args.catalog)
obs = Observation(config=config, Catalog=catalog_module.Catalog, work_dir=config['work_dir'], data_dir=config['data_dir'])
obs = Observation(config=config, Catalog=catalog_module.Catalog,
work_dir=config['work_dir'], data_dir=config['data_dir'])
else:
catalog_module = None
obs = Observation(config=config, Catalog=None, work_dir=config['work_dir'], data_dir=config['data_dir'])
obs = Observation(config=config, Catalog=None,
work_dir=config['work_dir'], data_dir=config['data_dir'])
# Run simulation
obs.runExposure_MPI_PointingList(pointing_list=pointing_list)
if __name__=='__main__':
if __name__ == '__main__':
run_sim()
setup.py
View file @ 3a9baf79
......@@ -16,7 +16,10 @@ from Cython.Build import cythonize
import numpy
class CTypes(Extension): pass
class CTypes(Extension):
pass
class build_ext(build_ext):
......@@ -34,24 +37,29 @@ class build_ext(build_ext):
return ext_name + '.so'
return super().get_ext_filename(ext_name)
extensions = [
Extension("ObservationSim.MockObject.SpecDisperser.disperse_c.interp", ["ObservationSim/MockObject/SpecDisperser/disperse_c/interp.pyx"],
include_dirs = [numpy.get_include()],
libraries=["m"]),
include_dirs=[numpy.get_include()],
libraries=["m"]),
Extension("ObservationSim.MockObject.SpecDisperser.disperse_c.disperse", ["ObservationSim/MockObject/SpecDisperser/disperse_c/disperse.pyx"],
include_dirs = [numpy.get_include()],
libraries=["m"]),
include_dirs=[numpy.get_include()],
libraries=["m"]),
]
df_module = [CTypes('ObservationSim.Instrument.Chip.libBF.libmoduleBF',
['ObservationSim/Instrument/Chip/libBF/diffusion_X1.c', 'ObservationSim/Instrument/Chip/libBF/nrutil.c'],
include_dirs=['ObservationSim/Instrument/Chip/libBF/', '/usr/include']
)]
['ObservationSim/Instrument/Chip/libBF/diffusion_X1.c',
'ObservationSim/Instrument/Chip/libBF/nrutil.c'],
include_dirs=[
'ObservationSim/Instrument/Chip/libBF/', '/usr/include']
)]
cti_module = [CTypes('ObservationSim.Instrument.Chip.libCTI.libmoduleCTI',
['ObservationSim/Instrument/Chip/libCTI/src/add_CTI.c', 'ObservationSim/Instrument/Chip/libCTI/src/nrutil.c', 'ObservationSim/Instrument/Chip/libCTI/src/ran1.c', 'ObservationSim/Instrument/Chip/libCTI/src/ran2.c', 'ObservationSim/Instrument/Chip/libCTI/src/poidev.c', 'ObservationSim/Instrument/Chip/libCTI/src/gammln.c', 'ObservationSim/Instrument/Chip/libCTI/src/gasdev.c', 'ObservationSim/Instrument/Chip/libCTI/src/sort.c', 'ObservationSim/Instrument/Chip/libCTI/src/creattraps.c'],
include_dirs=['ObservationSim/Instrument/Chip/libCTI/src/', '/usr/include']
)]
['ObservationSim/Instrument/Chip/libCTI/src/add_CTI.c', 'ObservationSim/Instrument/Chip/libCTI/src/nrutil.c', 'ObservationSim/Instrument/Chip/libCTI/src/ran1.c', 'ObservationSim/Instrument/Chip/libCTI/src/ran2.c', 'ObservationSim/Instrument/Chip/libCTI/src/poidev.c',
'ObservationSim/Instrument/Chip/libCTI/src/gammln.c', 'ObservationSim/Instrument/Chip/libCTI/src/gasdev.c', 'ObservationSim/Instrument/Chip/libCTI/src/sort.c', 'ObservationSim/Instrument/Chip/libCTI/src/creattraps.c'],
include_dirs=[
'ObservationSim/Instrument/Chip/libCTI/src/', '/usr/include']
)]
# setup(
......@@ -67,45 +75,45 @@ with open("requirements.txt", "r") as f:
if not req.startswith("#") and req.__contains__("==")
]
setup(name='CSSTSim',
version='2.1.0',
packages=find_packages(),
# install_requires=[
# # 'numpy>=1.18.5',
# # 'galsim>=2.2.4',
# # 'pyyaml>=5.3.1',
# # 'astropy>=4.0.1',
# # 'scipy>=1.5.0',
# # 'mpi4py>=3.0.3',
# # 'sep>=1.0.3',
# # 'healpy>=1.14.0',
# # 'h5py>=2.10.0',
# # 'Cython>=0.29.21',
# # 'numba>=0.50.1'
# ],
package_data = {
'ObservationSim.Astrometry.lib': ['libshao.so'],
'ObservationSim.Instrument.Chip.libBF': ['libmoduleBF.so'],
'ObservationSim.Instrument.Chip.libCTI': ['libmoduleCTI.so'],
'ObservationSim.MockObject.data': ['*.dat'],
'ObservationSim.MockObject.data.led': ['*.fits'],
'ObservationSim.Instrument.data': ['*.txt', '*.dat', '*.json'],
'ObservationSim.Instrument.data.field_distortion': ['*.pickle'],
'ObservationSim.Instrument.data.ccd': ['*.txt','*.json'],
'ObservationSim.Instrument.data.filters': ['*.txt', '*.list', '*.dat'],
'ObservationSim.Instrument.data.throughputs': ['*.txt', '*.dat'],
'ObservationSim.Instrument.data.sls_conf': ['*.conf', '*.fits'],
'ObservationSim.Instrument.data.flatCube': ['*.fits'],
'Catalog.data': ['*.fits','*.so'],
'ObservationSim.Config.Header':['*.fits','*.lst'],
'ObservationSim.Straylight.data': ['*.dat'],
'ObservationSim.Straylight.data.sky': ['*.dat'],
'ObservationSim.Straylight.lib': ['*'],
},
python_requires=">=3.11", # Python版本要求
install_requires=requirements,
ext_modules = cythonize(extensions) + df_module + cti_module,
cmdclass={'build_ext': build_ext}
)
setup(name='csst_msc_sim',
version='3.0.0',
packages=find_packages(),
# install_requires=[
# # 'numpy>=1.18.5',
# # 'galsim>=2.2.4',
# # 'pyyaml>=5.3.1',
# # 'astropy>=4.0.1',
# # 'scipy>=1.5.0',
# # 'mpi4py>=3.0.3',
# # 'sep>=1.0.3',
# # 'healpy>=1.14.0',
# # 'h5py>=2.10.0',
# # 'Cython>=0.29.21',
# # 'numba>=0.50.1'
# ],
package_data={
'ObservationSim.Astrometry.lib': ['libshao.so'],
'ObservationSim.Instrument.Chip.libBF': ['libmoduleBF.so'],
'ObservationSim.Instrument.Chip.libCTI': ['libmoduleCTI.so'],
'ObservationSim.MockObject.data': ['*.dat'],
'ObservationSim.MockObject.data.led': ['*.fits'],
'ObservationSim.Instrument.data': ['*.txt', '*.dat', '*.json'],
'ObservationSim.Instrument.data.field_distortion': ['*.pickle'],
'ObservationSim.Instrument.data.ccd': ['*.txt', '*.json'],
'ObservationSim.Instrument.data.filters': ['*.txt', '*.list', '*.dat'],
'ObservationSim.Instrument.data.throughputs': ['*.txt', '*.dat'],
'ObservationSim.Instrument.data.sls_conf': ['*.conf', '*.fits'],
'ObservationSim.Instrument.data.flatCube': ['*.fits'],
'Catalog.data': ['*.fits', '*.so'],
'ObservationSim.Config.Header': ['*.fits', '*.lst'],
'ObservationSim.Straylight.data': ['*.dat'],
'ObservationSim.Straylight.data.sky': ['*.dat'],
'ObservationSim.Straylight.lib': ['*'],
},
python_requires=">=3.11", # Python版本要求
install_requires=requirements,
ext_modules=cythonize(extensions) + df_module + cti_module,
cmdclass={'build_ext': build_ext}
)
test_C6.sh
View file @ 3a9baf79
......@@ -2,9 +2,9 @@
date
python3 /public/home/fangyuedong/project/csst-simulation/run_sim.py \
python3 /public/home/fangyuedong/project/csst_msc_sim/run_sim.py \
--config_file config_overall.yaml \
-c /public/home/fangyuedong/project/csst-simulation/config \
-c /public/home/fangyuedong/project/csst_msc_sim/config \
--catalog C6_50sqdeg
# python3 /share/home/fangyuedong/20231211/csst-simulation/run_sim.py \
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment