1. Hotfix: make the importlib.resources usages compatible with python3.9

2. Hotfix: fixed a bug when get the bandpasses for spectroscopic chips in Filter.py

ObservationSim/Astrometry/Astrometry_util.py

@@ -88,10 +88,14 @@ def on_orbit_obs_position(input_ra_list, input_dec_list, input_pmra_list, input_
                 raise TypeError("Parameter 16 minute range error [0 ~ 59]!", input_minute)
             if not (input_second>=0 and input_second<60.0):
                 raise TypeError("Parameter 16 second range error [0 ~ 60)!", input_second)
+    
     #Inital dynamic lib
-    # shao = cdll.LoadLibrary(lib_path)
-    with pkg_resources.path('ObservationSim.Astrometry.lib', "libshao.so") as lib_path:
-        shao = cdll.LoadLibrary(lib_path)
+    try:
+        with pkg_resources.files('ObservationSim.Astrometry.lib').joinpath("libshao.so") as lib_path:
+            shao = cdll.LoadLibrary(lib_path)
+    except AttributeError:
+        with pkg_resources.path('ObservationSim.Astrometry.lib', "libshao.so") as lib_path:
+            shao = cdll.LoadLibrary(lib_path)
     shao.onOrbitObs.restype = c_int
     
     d3 = c_double * 3

ObservationSim/Instrument/Chip/Chip.py

@@ -49,20 +49,29 @@ class Chip(FocalPlane):
         self.bound = self.getChipLim()
         self.ccdEffCurve_dir = ccdEffCurve_dir
         self.CRdata_dir = CRdata_dir
-        # self.sls_dir=sls_dir
-        # self.sls_conf = os.path.join(self.sls_dir, self.getChipSLSConf())
         slsconfs = self.getChipSLSConf()
         if np.size(slsconfs) == 1:
-            # self.sls_conf = [os.path.join(self.sls_dir, slsconfs)]
-            with pkg_resources.path('ObservationSim.Instrument.data.sls_conf', slsconfs) as conf_path:
-                self.sls_conf = str(conf_path)
+            try:
+                with pkg_resources.files('ObservationSim.Instrument.data.sls_conf').joinpath(slsconfs) as conf_path:
+                    self.sls_conf = str(conf_path)
+            except AttributeError:
+                with pkg_resources.path('ObservationSim.Instrument.data.sls_conf', slsconfs) as conf_path:
+                    self.sls_conf = str(conf_path)
         else:
             # self.sls_conf = [os.path.join(self.sls_dir, slsconfs[0]), os.path.join(self.sls_dir, slsconfs[1])]
             self.sls_conf = []
-            with pkg_resources.path('ObservationSim.Instrument.data.sls_conf', slsconfs[0]) as conf_path:
-                self.sls_conf.append(str(conf_path))
-            with pkg_resources.path('ObservationSim.Instrument.data.sls_conf', slsconfs[1]) as conf_path:
-                self.sls_conf.append(str(conf_path))
+            try:
+                with pkg_resources.files('ObservationSim.Instrument.data.sls_conf').joinpath(slsconfs[0]) as conf_path:
+                    self.sls_conf.append(str(conf_path))
+            except AttributeError:
+                with pkg_resources.path('ObservationSim.Instrument.data.sls_conf', slsconfs[0]) as conf_path:
+                    self.sls_conf.append(str(conf_path))
+            try:
+                with pkg_resources.files('ObservationSim.Instrument.data.sls_conf').joinpath(slsconfs[1]) as conf_path:
+                    self.sls_conf.append(str(conf_path))
+            except AttributeError:
+                with pkg_resources.path('ObservationSim.Instrument.data.sls_conf', slsconfs[1]) as conf_path:
+                    self.sls_conf.append(str(conf_path))
         
         self.effCurve = self._getChipEffCurve(self.filter_type)
         self._getCRdata()
@@ -103,8 +112,12 @@ class Chip(FocalPlane):
         
         # path = os.path.join(self.ccdEffCurve_dir, filename)
         # table = Table.read(path, format='ascii')
-        with pkg_resources.path('ObservationSim.Instrument.data.ccd', filename) as ccd_path:
-            table = Table.read(ccd_path, format='ascii')
+        try:
+            with pkg_resources.files('ObservationSim.Instrument.data.ccd').joinpath(filename) as ccd_path:
+                table = Table.read(ccd_path, format='ascii')
+        except AttributeError:
+            with pkg_resources.path('ObservationSim.Instrument.data.ccd', filename) as ccd_path:
+                table = Table.read(ccd_path, format='ascii')
         # throughput = galsim.LookupTable(x=table['col1'], f=table['col2']*mirror_eff, interpolant='linear')
         throughput = galsim.LookupTable(x=table['col1'], f=table['col2'], interpolant='linear')
         bandpass = galsim.Bandpass(throughput, wave_type='nm')
@@ -113,8 +126,12 @@ class Chip(FocalPlane):
     def _getCRdata(self):
         # path = os.path.join(self.CRdata_dir, 'wfc-cr-attachpixel.dat')
         # self.attachedSizes = np.loadtxt(path)
-        with pkg_resources.path('ObservationSim.Instrument.data', "wfc-cr-attachpixel.dat") as cr_path:
-            self.attachedSizes = np.loadtxt(cr_path)
+        try:
+            with pkg_resources.files('ObservationSim.Instrument.data').joinpath("wfc-cr-attachpixel.dat") as cr_path:
+                self.attachedSizes = np.loadtxt(cr_path)
+        except AttributeError:
+            with pkg_resources.path('ObservationSim.Instrument.data', "wfc-cr-attachpixel.dat") as cr_path:
+                self.attachedSizes = np.loadtxt(cr_path)
 
     def getChipFilter(self, chipID=None, filter_layout=None):
         """Return the filter index and type for a given chip #(chipID)

ObservationSim/Instrument/Filter.py

@@ -51,18 +51,28 @@ class Filter(object):
             # Get full-bandpass
             # filter_file = os.path.join(filter_dir, self.filter_type+".dat")
             # bandpass_full = galsim.Bandpass(filter_file, wave_type=unit)
-            with pkg_resources.path('ObservationSim.Instrument.data.filters', self.filter_type.lower() + '.txt') as filter_file:
-                self.filter_bandpass = galsim.Bandpass(str(filter_file), wave_type=unit)
-            with pkg_resources.path('ObservationSim.Instrument.data.throughputs', self.filter_type.lower() + '_throughput.txt') as filter_file:
-                bandpass_full = galsim.Bandpass(str(filter_file), wave_type=unit)
+            try:
+                with pkg_resources.files('ObservationSim.Instrument.data.filters').joinpath(self.filter_type.lower() + '.txt') as filter_file:
+                    self.filter_bandpass = galsim.Bandpass(str(filter_file), wave_type=unit)
+            except AttributeError:
+                with pkg_resources.path('ObservationSim.Instrument.data.filters', self.filter_type.lower() + '.txt') as filter_file:
+                    self.filter_bandpass = galsim.Bandpass(str(filter_file), wave_type=unit)
+            try:
+                with pkg_resources.files('ObservationSim.Instrument.data.throughputs').joinpath(self.filter_type.lower() + '_throughput.txt') as filter_file:
+                    bandpass_full = galsim.Bandpass(str(filter_file), wave_type=unit)
+            except AttributeError:
+                with pkg_resources.path('ObservationSim.Instrument.data.throughputs', self.filter_type.lower() + '_throughput.txt') as filter_file:
+                    bandpass_full = galsim.Bandpass(str(filter_file), wave_type=unit)
             # bandpass_full = bandpass_full * self.ccd_bandpass
 
             # Get sub-bandpasses
             bandpass_sub_list = []
-            # wave_bin_file = os.path.join(filter_dir, self.filter_type.lower() + "_sub.list")
-            # wave_points = open(wave_bin_file).read().splitlines()
-            with pkg_resources.path('ObservationSim.Instrument.data.filters', self.filter_type.lower() + "_sub.list") as wave_bin_file:
-                wave_points = open(wave_bin_file).read().splitlines()
+            try:
+                with pkg_resources.files('ObservationSim.Instrument.data.filters').joinpath(self.filter_type.lower() + "_sub.list") as wave_bin_file:
+                    wave_points = open(wave_bin_file).read().splitlines()
+            except AttributeError:
+                with pkg_resources.path('ObservationSim.Instrument.data.filters', self.filter_type.lower() + "_sub.list") as wave_bin_file:
+                    wave_points = open(wave_bin_file).read().splitlines()
 
             for i in range(2, len(wave_points), 2):
                 blim = max(float(wave_points[i-2])*0.1, bandpass_full.blue_limit)
@@ -75,13 +85,15 @@ class Filter(object):
         else:   # Spectroscopic
             sls_lamb = np.linspace(self.blue_limit, self.red_limit, 100)
             sls_flux = np.ones_like(sls_lamb)
-            con_spec = galsim.LookupTable(sls_lamb, sls_lamb, interpolant='nearest')
+            con_spec = galsim.LookupTable(sls_lamb, sls_flux, interpolant='nearest')
             bandpass_full = galsim.Bandpass(con_spec, wave_type=unit)
             bandpass_sub_list = []
-            # wave_bin_file = os.path.join(filter_dir, self.filter_type.lower() + "_sub.list")
-            # wave_points = open(wave_bin_file).read().splitlines()
-            with pkg_resources.path('ObservationSim.Instrument.data.filters', self.filter_type.lower() + "_sub.list") as wave_bin_file:
-                wave_points = open(wave_bin_file).read().splitlines()
+            try:
+                with pkg_resources.files('ObservationSim.Instrument.data.filters').joinpath(self.filter_type.lower() + "_sub.list") as wave_bin_file:
+                    wave_points = open(wave_bin_file).read().splitlines()
+            except AttributeError:
+                with pkg_resources.path('ObservationSim.Instrument.data.filters', self.filter_type.lower() + "_sub.list") as wave_bin_file:
+                    wave_points = open(wave_bin_file).read().splitlines()
             for i in range(2, len(wave_points), 2):
                 blim = max(float(wave_points[i - 2]) * 0.1, bandpass_full.blue_limit)
                 rlim = min(float(wave_points[i]) * 0.1, bandpass_full.red_limit)

ObservationSim/Instrument/Telescope.py

@@ -15,8 +15,12 @@ class Telescope(object):
 		if optEffCurve_path is not None:
 			self.efficiency = self._get_efficiency(optEffCurve_path)
 		else:
-			with pkg_resources.path('ObservationSim.Instrument.data', 'mirror_ccdnote.txt') as optEffCurve_path:
-				self.efficiency = self._get_efficiency(optEffCurve_path)
+			try:
+				with pkg_resources.files('ObservationSim.Instrument.data').joinpath('mirror_ccdnote.txt') as optEffCurve_path:
+					self.efficiency = self._get_efficiency(optEffCurve_path)
+			except AttributeError:
+				with pkg_resources.path('ObservationSim.Instrument.data', 'mirror_ccdnote.txt') as optEffCurve_path:
+					self.efficiency = self._get_efficiency(optEffCurve_path)
 
 	def _get_efficiency(self, effCurve_path):
 		""" Read in the efficiency of optics

ObservationSim/Instrument/_util.py

@@ -43,8 +43,12 @@ return {*} limit mag and saturation mag
 '''
 
 def calculateLimitMag(aperture = 2.0, psf_fwhm = 0.1969,pixelSize = 0.074, pmRation = 0.8, throughputFn = 'i_throughput.txt', readout = 5.0, skyFn= 'sky_emiss_hubble_50_50_A.dat', darknoise = 0.02,exTime = 150, exNum = 1, fw = 90000):
-	with pkg_resources.path('ObservationSim.Instrument.data.throughputs', throughputFn) as data_file:
-		throughput_f = np.loadtxt(data_file)
+	try:
+		with pkg_resources.files('ObservationSim.Instrument.data.throughputs').joinpath(throughputFn) as data_file:
+			throughput_f = np.loadtxt(data_file)
+	except AttributeError:
+		with pkg_resources.path('ObservationSim.Instrument.data.throughputs', throughputFn) as data_file:
+			throughput_f = np.loadtxt(data_file)
 	thr_i = interpolate.interp1d(throughput_f[:,0]/10, throughput_f[:,1]); # wavelength in anstrom
 	f_s = 200
 	f_e = 1100
@@ -59,8 +63,12 @@ def calculateLimitMag(aperture = 2.0, psf_fwhm = 0.1969,pixelSize = 0.074, pmRat
 	wave = np.arange(f_s,f_e+delt_f,delt_f)
 	wavey = np.ones(wave.shape[0])
 
-	with pkg_resources.path('ObservationSim.Instrument.data.throughputs', skyFn) as data_file:
-		skydata = np.loadtxt(data_file)
+	try:
+		with pkg_resources.files('ObservationSim.Instrument.data.throughputs').joinpath(skyFn) as data_file:
+			skydata = np.loadtxt(data_file)
+	except AttributeError:
+		with pkg_resources.path('ObservationSim.Instrument.data.throughputs', skyFn) as data_file:
+			skydata = np.loadtxt(data_file)
 	skydatai = interpolate.interp1d(skydata[:,0]/10, skydata[:,1]*10)
 
 	sky_data = np.zeros([data_num,2])

ObservationSim/MockObject/SkybackgroundMap.py

@@ -41,9 +41,12 @@ def calculateSkyMap_split_g(skyMap=None, blueLimit=4200, redLimit=6500, skyfn='s
     fimg = np.zeros_like(skyMap)
 
     fImg = galsim.Image(fimg)
-    # skyfn = os.path.join(SLSSIM_PATH, skyfn)
-    with pkg_resources.path('ObservationSim.MockObject.data', skyfn) as data_path:
-        skySpec = np.loadtxt(data_path)
+    try:
+        with pkg_resources.files('ObservationSim.MockObject.data').joinpath(skyfn) as data_path:
+            skySpec = np.loadtxt(data_path)
+    except AttributeError:
+        with pkg_resources.path('ObservationSim.MockObject.data', skyfn) as data_path:
+            skySpec = np.loadtxt(data_path)
     # skySpec = np.loadtxt(skyfn)
     spec = Table(np.array([skySpec[:, 0], skySpec[:, 1]]).T, names=('WAVELENGTH', 'FLUX'))
 
@@ -154,8 +157,12 @@ def calculateSkyMap(xLen=9232, yLen=9126, blueLimit=4200, redLimit=6500,
 
     fimg = np.zeros_like(skyMap)
     fImg = galsim.Image(fimg)
-    with pkg_resources.path('ObservationSim.MockObject.data', skyfn) as data_path:
-        skySpec = np.loadtxt(data_path)
+    try:
+        with pkg_resources.files('ObservationSim.MockObject.data').joinpath(skyfn) as data_path:
+            skySpec = np.loadtxt(data_path)
+    except AttributeError:
+        with pkg_resources.path('ObservationSim.MockObject.data', skyfn) as data_path:
+            skySpec = np.loadtxt(data_path)
     # skySpec = np.loadtxt(skyfn)
     
     spec = Table(np.array([skySpec[:, 0], skySpec[:, 1]]).T, names=('WAVELENGTH', 'FLUX'))

config/config_NGP_dev.yaml

@@ -10,18 +10,16 @@
 # Can add some of the command-line arguments here as well;
 # OK to pass either way or both, as long as they are consistent
 # work_dir: "/public/home/fangyuedong/sim_code_release/CSST/test/"
-work_dir: "/public/home/fangyuedong/test/CSST/workplace/"
-data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
+work_dir: "/share/home/fangyuedong/csst-simulation/workplace/"
+data_dir: "/share/simudata/CSSOSDataProductsSims/data/"
 run_name: "v0.5_TEST"
 
 # (Optional) a file of point list 
 # if you just want to run default pointing:
 # - pointing_dir: null
 # - pointing_file: null
-# pointing_dir: "/data/simudata/CSSOSDataProductsSims/data/"
-pointing_dir: "/public/home/fangyuedong/test/CSST/test_20220622/"
+pointing_dir: "/share/simudata/CSSOSDataProductsSims/data/"
 pointing_file: "pointing_test_NGP_2.17.dat"
-# pointing_file: "pointing_test_case2.dat"
 
 # Whether to use MPI
 run_option:
@@ -127,7 +125,7 @@ psf_setting:
 
   # path to PSF data
   # NOTE: only valid for "Interp" PSF
-  psf_dir: "/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube"
+  psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube"
 
   # path to field-distortion model
   # Note: only valid when ins_effects: field_dist is "ON"

run_NGP.pbs

 #!/bin/bash
 
 #PBS -N SIMS
-##PBS -l walltime=70:00:00
-
-##mpdallexit
-##mpdboot -n 10 -f  ./mpd.hosts
-
-##PBS -j oe
-#PBS -l nodes=comput110:ppn=80
-#####PBS -q longq
-#PBS -q batch
+#PBS -l nodes=wcl-1:ppn=32+wcl-6:ppn=32
 #PBS -u fangyuedong
+###PBS -j oe
 
-NP=80
+cd $PBS_O_WORKDIR
+
+NP=32
 date
 echo $NP
 
 # mpirun -np $NP python /public/home/fangyuedong/test/CSST/run_sim.py config_NGP.yaml -c /public/home/fangyuedong/test/CSST/config
-mpirun -np $NP python /public/home/fangyuedong/test/CSST/run_sim.py config_NGP_dev.yaml -c /public/home/fangyuedong/test/CSST/config
\ No newline at end of file
+mpirun -np $NP python3 /share/home/fangyuedong/csst-simulation/run_sim.py config_NGP_dev.yaml -c /share/home/fangyuedong/csst-simulation/config
+

setup.py

@@ -28,19 +28,19 @@ extensions = [
 setup(name='CSSTSim', 
     version='0.5.2', 
     packages=find_packages(),
-    # install_requires=[
-    #     'numpy>=1.18.5',
-    #     'galsim>=2.2.4',
-    #     'pyyaml>=5.3.1',
-    #     'astropy>=4.0.1',
-    #     'scipy>=1.5.0',
-    #     'mpi4py>=3.0.3',
-    #     'sep>=1.0.3',
-    #     'healpy>=1.14.0',
-    #     'h5py>=2.10.0',
-    #     'Cython>=0.29.21'
-    #     'numba>=0.50.1'
-    # ],
+    install_requires=[
+        'numpy>=1.18.5',
+        'galsim>=2.2.4',
+        'pyyaml>=5.3.1',
+        'astropy>=4.0.1',
+        'scipy>=1.5.0',
+        'mpi4py>=3.0.3',
+        'sep>=1.0.3',
+        'healpy>=1.14.0',
+        'h5py>=2.10.0',
+        'Cython>=0.29.21'
+        'numba>=0.50.1'
+    ],
     package_data = {
         'ObservationSim.Astrometry.lib': ['libshao.so'], 
         'ObservationSim.MockObject.data': ['*.dat'],