Merge branch 'sim_scheduler' into 'develop'

package name and version change

See merge request csst-sims/csst_msc_sim!19

.gitignore

 build/*
-CSSTSim.egg-info/*
+csst_msc_sim.egg-info/*
 dist/*
 *.pyc
 *.so

Catalog/Catalog_example.py

@@ -53,7 +53,7 @@ class Catalog(CatalogBase):
         super().__init__()
         self.cat_dir = os.path.join(config["data_dir"], config["catalog_options"]["input_path"]["cat_dir"])
         self.chip = chip
-        if "star_cat" in config["catalog_options"]["input_path"] and config["catalog_options"]["input_path"]["star_cat"]
+        if "star_cat" in config["catalog_options"]["input_path"] and config["catalog_options"]["input_path"]["star_cat"]:
             star_file = config["catalog_options"]["input_path"]["star_cat"]
             star_SED_file = config["catalog_options"]["SED_templates_path"]["star_SED"]
             self.star_path = os.path.join(self.cat_dir, star_file)

ObservationSim/Config/Header/ImageHeader.py

@@ -473,7 +473,7 @@ def generatePrimaryHeader(xlen = 9216, ylen = 9232, pointing_id = '00000001', po
     # h_prim['SIM_VER'] = (get_distribution("CSSTSim").version, "Version of CSST MSC simulation software")
     currentDateAndTime = datetime.now()
     compute_name = platform.node()
-    h_prim['FITSSWV'] = get_distribution("CSSTSim").version +'_' + currentDateAndTime.strftime("%Y%m%d") + '_' +compute_name
+    h_prim['FITSSWV'] = get_distribution("csst_msc_sim").version +'_' + currentDateAndTime.strftime("%Y%m%d") + '_' +compute_name
     h_prim['EPOCH'] = round((Time(h_prim['EXPSTART'], format='mjd', scale='tcb')).jyear, 1)
 
     return h_prim

ObservationSim/MockObject/FlatLED.py

@@ -52,7 +52,7 @@ fluxLED = {'LED1': 15, 'LED2': 15, 'LED3': 12.5, 'LED4': 9, 'LED5': 9,
 mirro_eff = {'GU':0.61, 'GV':0.8, 'GI':0.8}
 # mirro_eff = {'GU':1, 'GV':1, 'GI':1}
 
-class FlatLED(object):
+class FlatLED(MockObject):
     def __init__(self, chip,filt, flatDir = None, logger=None):
         # self.led_type_list = led_type_list
         self.filt = filt
@@ -426,7 +426,7 @@ class FlatLED(object):
                     print('time: %s ms' % ((T2 - T1) * 1000))
 
         if isAlongY == 1:
-            fimg, tmx, tmy = rotate90(array_orig=fImg.array, xc=0, yc=0, isClockwise=0)
+            fimg, tmx, tmy = SpecDisperser.rotate90(array_orig=fImg.array, xc=0, yc=0, isClockwise=0)
         else:
             fimg = fImg.array
 

ObservationSim/sim_steps/add_LED_flat.py

@@ -29,7 +29,7 @@ def add_LED_Flat(self, chip, filt, tel, pointing, catalog, obs_param):
         pf_map = np.array(pf_map, dtype='float32')
         self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True])
     else:
-        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [True,'','','',''])
+        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN1','SHTCLOS0'], values = [True,self.h_ext['SHTCLOS1'],self.h_ext['SHTOPEN0']])
 
     chip.img = chip.img + pf_map
 

ObservationSim/sim_steps/add_objects.py

@@ -211,7 +211,7 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param):
         flat_normal = np.array(flat_normal, dtype='float32')
         self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True])
     else:
-        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [True,'','','',''])
+        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN1','SHTCLOS0'], values = [True,self.h_ext['SHTCLOS1'],self.h_ext['SHTOPEN0']])
     chip.img *= flat_normal
     del flat_normal
 

ObservationSim/sim_steps/add_sky_background.py

@@ -22,7 +22,7 @@ def add_sky_background_sci(self, chip, filt, tel, pointing, catalog, obs_param):
         flat_normal = np.array(flat_normal, dtype='float32')
         self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True])
     else:
-        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [False,'','','',''])
+        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN1','SHTCLOS0'], values = [True,self.h_ext['SHTCLOS1'],self.h_ext['SHTOPEN0']])
     
     if obs_param["enable_straylight_model"]:
         # Filter.sky_background, Filter.zodical_spec will be updated
@@ -84,7 +84,7 @@ def add_sky_flat_calibration(self, chip, filt, tel, pointing, catalog, obs_param
             shutt_gsimg.write("%s/ShutterEffect_%s_1.fits" % (self.chip_output.subdir, str(chip.chipID).rjust(2, '0')))
         self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True])
     else:
-        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [True,'','','',''])
+        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN1','SHTCLOS0'], values = [True,self.h_ext['SHTCLOS1'],self.h_ext['SHTOPEN0']])
     
 
     if chip.survey_type == "photometric":

ObservationSim/sim_steps/readout_output.py

@@ -45,7 +45,7 @@ def quantization_and_output(self, chip, filt, tel, pointing, catalog, obs_param)
 
     if not hasattr(self, 'h_ext'):
         _, _ = self.prepare_headers(chip=chip, pointing=pointing)
-        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1','EXPTIME'], values = [False,'','','','',0.0])
+        self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN1','SHTCLOS0','SHTCLOS1','EXPTIME'], values = [False,self.h_ext['SHTOPEN0'],self.h_ext['SHTOPEN0'],self.h_ext['SHTOPEN0'],0.0])
         # renew header info
         datetime_obs = datetime.utcfromtimestamp(pointing.timestamp)
         datetime_obs = datetime_obs.replace(tzinfo=timezone.utc)

config/config_overall.yaml

@@ -46,6 +46,9 @@ catalog_options:
   # Only simulate galaxies?
   galaxy_only: YES
 
+  # rotate galaxy ellipticity
+  rotateEll: 0. # [degree]
+
 ###############################################
 # Observation setting
 ###############################################
@@ -56,7 +59,7 @@ obs_setting:
   # - pointing_file: null
   pointing_file: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/pointing50_C9/pointing_50_1_n.dat"
 
-  obs_config_file: "/public/home/fangyuedong/project/csst-simulation/config/obs_config_SCI_WIDE_phot.yaml"
+  obs_config_file: "/public/home/fangyuedong/project/csst_msc_sim/config/obs_config_SCI_WIDE_phot.yaml"
 
   # Run specific pointing(s):
   # - give a list of indexes of pointings: [ip_1, ip_2...]

config/config_overall_newStar.yaml

@@ -46,6 +46,9 @@ catalog_options:
   # Only simulate galaxies?
   galaxy_only: NO
 
+  # rotate galaxy ellipticity
+  rotateEll: 0. # [degree]
+
 ###############################################
 # Observation setting
 ###############################################

install.sh

@@ -5,5 +5,5 @@ conda install numpy==1.26.4
 conda install cython==3.0.6
 conda install sep==1.2.1
 conda install mpi4py==3.1.6
-python3 setup.py install --user
+conda install cfitsio==3.470
 pip install -e .

run_sim.py

@@ -10,6 +10,7 @@ import importlib
 import gc
 gc.enable()
 
+
 def run_sim():
     """
     Main method for simulation call.
@@ -18,13 +19,13 @@ def run_sim():
     ----------
     Catalog : Class
         a catalog class which is inherited from ObservationSim.MockObject.CatalogBase 
-    
+
     Returns
     ----------
         None
     """
-    # Get version of CSSTSim Package
-    __version__ = version("CSSTSim")
+    # Get version of the Package
+    __version__ = version("csst_msc_sim")
 
     # Get run datetime
     now = datetime.datetime.now()
@@ -38,7 +39,7 @@ def run_sim():
         try:
             config = yaml.safe_load(stream)
             for key, value in config.items():
-                print (key + " : " + str(value))
+                print(key + " : " + str(value))
         except yaml.YAMLError as exc:
             print(exc)
 
@@ -62,42 +63,48 @@ def run_sim():
     if "run_counter" not in config:
         config["run_counter"] = 0
 
-    # Generate lists pointings based on the input pointing list (or default 
+    # Generate lists pointings based on the input pointing list (or default
     # pointing RA, DEC) and "config["obs_setting"]["run_pointings"]".
-    # "config['obs_setting']['np_cal']"" is the number of CAL pointings which will be 
+    # "config['obs_setting']['np_cal']"" is the number of CAL pointings which will be
     # appended to the front.
     # NOTE: the implementation of gerenating time_stamps is temporary.
     pointing_dir = None
     if "pointing_dir" in config['obs_setting']:
         pointing_dir = config['obs_setting']["pointing_dir"]
-    pointing_list = generate_pointing_list(config=config, pointing_filename=config['obs_setting']['pointing_file'], data_dir=pointing_dir)
+    pointing_list = generate_pointing_list(
+        config=config, pointing_filename=config['obs_setting']['pointing_file'], data_dir=pointing_dir)
 
     # Make the main output directories
-    run_dir = make_run_dirs(work_dir=config['work_dir'], run_name=config['run_name'], pointing_list=pointing_list)
-    
+    run_dir = make_run_dirs(
+        work_dir=config['work_dir'], run_name=config['run_name'], pointing_list=pointing_list)
+
     # Copy the config file to output directory & Write Run metadata
     shutil.copy(args.config_file, run_dir)
     run_meta = os.path.join(run_dir, "run_metadata.yaml")
     with open(run_meta, "w") as config_out:
         config_out.write("\n")
         config_out.write("###############################################\n")
-        config_out.write("CSSTSim_version: \"%s\"\n"%__version__)
+        config_out.write(
+            "csst_msc_sim package version: \"%s\"\n" % __version__)
         date_str = datetime.datetime.strftime(now, '%m/%d/%Y')
         time_str = datetime.datetime.strftime(now, '%H:%M:%S')
-        config_out.write("Run_date: \"%s\"\n"%date_str)
-        config_out.write("Run_time: \"%s\"\n"%time_str)
+        config_out.write("Run_date: \"%s\"\n" % date_str)
+        config_out.write("Run_time: \"%s\"\n" % time_str)
         config_out.write("###############################################\n")
 
     # Initialize the simulation
     if args.catalog is not None:
         catalog_module = importlib.import_module('Catalog.'+args.catalog)
-        obs = Observation(config=config, Catalog=catalog_module.Catalog, work_dir=config['work_dir'], data_dir=config['data_dir'])
+        obs = Observation(config=config, Catalog=catalog_module.Catalog,
+                          work_dir=config['work_dir'], data_dir=config['data_dir'])
     else:
         catalog_module = None
-        obs = Observation(config=config, Catalog=None, work_dir=config['work_dir'], data_dir=config['data_dir'])
-    
+        obs = Observation(config=config, Catalog=None,
+                          work_dir=config['work_dir'], data_dir=config['data_dir'])
+
     # Run simulation
     obs.runExposure_MPI_PointingList(pointing_list=pointing_list)
 
-if __name__=='__main__':
+
+if __name__ == '__main__':
     run_sim()

setup.py

@@ -16,7 +16,10 @@ from Cython.Build import cythonize
 
 import numpy
 
-class CTypes(Extension): pass
+
+class CTypes(Extension):
+    pass
+
 
 class build_ext(build_ext):
 
@@ -34,24 +37,29 @@ class build_ext(build_ext):
             return ext_name + '.so'
         return super().get_ext_filename(ext_name)
 
+
 extensions = [
     Extension("ObservationSim.MockObject.SpecDisperser.disperse_c.interp", ["ObservationSim/MockObject/SpecDisperser/disperse_c/interp.pyx"],
-        include_dirs = [numpy.get_include()],
-        libraries=["m"]),
-    
+              include_dirs=[numpy.get_include()],
+              libraries=["m"]),
+
     Extension("ObservationSim.MockObject.SpecDisperser.disperse_c.disperse", ["ObservationSim/MockObject/SpecDisperser/disperse_c/disperse.pyx"],
-        include_dirs = [numpy.get_include()],
-        libraries=["m"]),
+              include_dirs=[numpy.get_include()],
+              libraries=["m"]),
 ]
 
 df_module = [CTypes('ObservationSim.Instrument.Chip.libBF.libmoduleBF',
-                    ['ObservationSim/Instrument/Chip/libBF/diffusion_X1.c', 'ObservationSim/Instrument/Chip/libBF/nrutil.c'],
-                    include_dirs=['ObservationSim/Instrument/Chip/libBF/', '/usr/include']
-)]
+                    ['ObservationSim/Instrument/Chip/libBF/diffusion_X1.c',
+                        'ObservationSim/Instrument/Chip/libBF/nrutil.c'],
+                    include_dirs=[
+                        'ObservationSim/Instrument/Chip/libBF/', '/usr/include']
+                    )]
 cti_module = [CTypes('ObservationSim.Instrument.Chip.libCTI.libmoduleCTI',
-                    ['ObservationSim/Instrument/Chip/libCTI/src/add_CTI.c', 'ObservationSim/Instrument/Chip/libCTI/src/nrutil.c', 'ObservationSim/Instrument/Chip/libCTI/src/ran1.c', 'ObservationSim/Instrument/Chip/libCTI/src/ran2.c', 'ObservationSim/Instrument/Chip/libCTI/src/poidev.c', 'ObservationSim/Instrument/Chip/libCTI/src/gammln.c', 'ObservationSim/Instrument/Chip/libCTI/src/gasdev.c', 'ObservationSim/Instrument/Chip/libCTI/src/sort.c', 'ObservationSim/Instrument/Chip/libCTI/src/creattraps.c'],
-                    include_dirs=['ObservationSim/Instrument/Chip/libCTI/src/', '/usr/include']
-)]
+                     ['ObservationSim/Instrument/Chip/libCTI/src/add_CTI.c', 'ObservationSim/Instrument/Chip/libCTI/src/nrutil.c', 'ObservationSim/Instrument/Chip/libCTI/src/ran1.c', 'ObservationSim/Instrument/Chip/libCTI/src/ran2.c', 'ObservationSim/Instrument/Chip/libCTI/src/poidev.c',
+                         'ObservationSim/Instrument/Chip/libCTI/src/gammln.c', 'ObservationSim/Instrument/Chip/libCTI/src/gasdev.c', 'ObservationSim/Instrument/Chip/libCTI/src/sort.c', 'ObservationSim/Instrument/Chip/libCTI/src/creattraps.c'],
+                     include_dirs=[
+                         'ObservationSim/Instrument/Chip/libCTI/src/', '/usr/include']
+                     )]
 
 
 # setup(
@@ -67,45 +75,45 @@ with open("requirements.txt", "r") as f:
         if not req.startswith("#") and req.__contains__("==")
     ]
 
-setup(name='CSSTSim', 
-    version='2.1.0', 
-    packages=find_packages(),
-    # install_requires=[
-    #     # 'numpy>=1.18.5',
-    #     # 'galsim>=2.2.4',
-    #     # 'pyyaml>=5.3.1',
-    #     # 'astropy>=4.0.1',
-    #     # 'scipy>=1.5.0',
-    #     # 'mpi4py>=3.0.3',
-    #     # 'sep>=1.0.3',
-    #     # 'healpy>=1.14.0',
-    #     # 'h5py>=2.10.0',
-    #     # 'Cython>=0.29.21',
-    #     # 'numba>=0.50.1'
-    # ],
-    
-    package_data = {
-        'ObservationSim.Astrometry.lib': ['libshao.so'], 
-        'ObservationSim.Instrument.Chip.libBF': ['libmoduleBF.so'],
-        'ObservationSim.Instrument.Chip.libCTI': ['libmoduleCTI.so'],
-        'ObservationSim.MockObject.data': ['*.dat'], 
-        'ObservationSim.MockObject.data.led': ['*.fits'],
-        'ObservationSim.Instrument.data': ['*.txt', '*.dat', '*.json'],
-        'ObservationSim.Instrument.data.field_distortion': ['*.pickle'],
-        'ObservationSim.Instrument.data.ccd': ['*.txt','*.json'], 
-        'ObservationSim.Instrument.data.filters': ['*.txt', '*.list', '*.dat'], 
-        'ObservationSim.Instrument.data.throughputs': ['*.txt', '*.dat'], 
-        'ObservationSim.Instrument.data.sls_conf': ['*.conf', '*.fits'],
-        'ObservationSim.Instrument.data.flatCube': ['*.fits'],
-        'Catalog.data': ['*.fits','*.so'],
-        'ObservationSim.Config.Header':['*.fits','*.lst'],
-        'ObservationSim.Straylight.data': ['*.dat'],
-        'ObservationSim.Straylight.data.sky': ['*.dat'],
-        'ObservationSim.Straylight.lib': ['*'],
-    },
-    python_requires=">=3.11",  # Python版本要求
-    install_requires=requirements,
-
-    ext_modules = cythonize(extensions) + df_module + cti_module,
-    cmdclass={'build_ext': build_ext}
-)
+setup(name='csst_msc_sim',
+      version='3.0.0',
+      packages=find_packages(),
+      # install_requires=[
+      #     # 'numpy>=1.18.5',
+      #     # 'galsim>=2.2.4',
+      #     # 'pyyaml>=5.3.1',
+      #     # 'astropy>=4.0.1',
+      #     # 'scipy>=1.5.0',
+      #     # 'mpi4py>=3.0.3',
+      #     # 'sep>=1.0.3',
+      #     # 'healpy>=1.14.0',
+      #     # 'h5py>=2.10.0',
+      #     # 'Cython>=0.29.21',
+      #     # 'numba>=0.50.1'
+      # ],
+
+      package_data={
+          'ObservationSim.Astrometry.lib': ['libshao.so'],
+          'ObservationSim.Instrument.Chip.libBF': ['libmoduleBF.so'],
+          'ObservationSim.Instrument.Chip.libCTI': ['libmoduleCTI.so'],
+          'ObservationSim.MockObject.data': ['*.dat'],
+          'ObservationSim.MockObject.data.led': ['*.fits'],
+          'ObservationSim.Instrument.data': ['*.txt', '*.dat', '*.json'],
+          'ObservationSim.Instrument.data.field_distortion': ['*.pickle'],
+          'ObservationSim.Instrument.data.ccd': ['*.txt', '*.json'],
+          'ObservationSim.Instrument.data.filters': ['*.txt', '*.list', '*.dat'],
+          'ObservationSim.Instrument.data.throughputs': ['*.txt', '*.dat'],
+          'ObservationSim.Instrument.data.sls_conf': ['*.conf', '*.fits'],
+          'ObservationSim.Instrument.data.flatCube': ['*.fits'],
+          'Catalog.data': ['*.fits', '*.so'],
+          'ObservationSim.Config.Header': ['*.fits', '*.lst'],
+          'ObservationSim.Straylight.data': ['*.dat'],
+          'ObservationSim.Straylight.data.sky': ['*.dat'],
+          'ObservationSim.Straylight.lib': ['*'],
+      },
+      python_requires=">=3.11",  # Python版本要求
+      install_requires=requirements,
+
+      ext_modules=cythonize(extensions) + df_module + cti_module,
+      cmdclass={'build_ext': build_ext}
+      )

test_C6.sh

@@ -2,9 +2,9 @@
 
 date
 
-python3 /public/home/fangyuedong/project/csst-simulation/run_sim.py \
+python3 /public/home/fangyuedong/project/csst_msc_sim/run_sim.py \
     --config_file config_overall.yaml \
-    -c /public/home/fangyuedong/project/csst-simulation/config \
+    -c /public/home/fangyuedong/project/csst_msc_sim/config \
     --catalog C6_50sqdeg 
 
 # python3 /share/home/fangyuedong/20231211/csst-simulation/run_sim.py \