v0 for testing

ObservationSim/MockObject/Galaxy.py

@@ -27,7 +27,7 @@ class Galaxy(MockObject):
         if rotation is not None:
             self.rotateEllipticity(rotation)
 
-    def load_SED(self, survey_type, sed_path, cosids=None, objtypes=None, sed_templates=None, normFilter=None, target_filt=None):
+    def load_SED(self, survey_type, sed_path=None, cosids=None, objtypes=None, sed_templates=None, normFilter=None, target_filt=None):
         if survey_type == "photometric":
             norm_thr_rang_ids = normFilter['SENSITIVITY'] > 0.001
             if sed_templates is None:

ObservationSim/MockObject/Quasar.py

@@ -12,7 +12,7 @@ class Quasar(MockObject):
     def __init__(self, param):
         super().__init__(param)
 
-    def load_SED(self, survey_type, sed_path, cosids=None, objtypes=None, sed_templates=None, normFilter=None, target_filt=None):
+    def load_SED(self, survey_type, sed_path=None, cosids=None, objtypes=None, sed_templates=None, normFilter=None, target_filt=None):
         if survey_type == "photometric":
             norm_thr_rang_ids = normFilter['SENSITIVITY'] > 0.001
             if sed_templates is None:

ObservationSim/ObservationSim.py

-from Config import ConfigDir, ReadConfig, ChipOutput
+from Config import ConfigDir, ChipOutput
 from Config.Header import generatePrimaryHeader, generateExtensionHeader
 from Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip
 from MockObject import Catalog, MockObject, Star, Galaxy, Quasar, calculateSkyMap_split_g
-from PSF import PSFGauss, PSFInterp, FieldDistortion
-from _util import makeSubDir, getShearFiled, makeSubDir_PointingList
+from PSF import PSFGauss, FieldDistortion, PSFInterp
+from _util import getShearFiled, makeSubDir_PointingList
 from astropy.io import fits
 from datetime import datetime
 import numpy as np
@@ -14,17 +14,18 @@ import logging
 import psutil
 
 class Observation(object):
-    def __init__(self, input_cat_dir=None, work_dir=None, data_dir=None):
-        self.path_dict = ConfigDir(input_cat_dir, work_dir, data_dir)
-        self.config = ReadConfig(self.path_dict["config_file"])
+    def __init__(self, config, work_dir=None, data_dir=None):
+        self.path_dict = ConfigDir(config=config, work_dir=work_dir, data_dir=data_dir)
+        # self.config = ReadConfig(self.path_dict["config_file"])
+        self.config = config
         self.tel = Telescope(optEffCurve_path=self.path_dict["mirror_file"]) # Currently the default values are hard coded in
-        self.focal_plane = FocalPlane(survey_type=self.config["survey_type"]) # Currently the default values are hard coded in
+        self.focal_plane = FocalPlane(survey_type=self.config["obs_setting"]["survey_type"]) # Currently the default values are hard coded in
         self.filter_param = FilterParam(filter_dir=self.path_dict["filter_dir"]) # Currently the default values are hard coded in
         self.chip_list = []
         self.filter_list = []
 
         # if we want to apply field distortion?
-        if self.config["field_dist"].lower() == "y":
+        if self.config["ins_effects"]["field_dist"] == True:
             self.fd_model = FieldDistortion()
         else:
             self.fd_model = None
@@ -47,17 +48,17 @@ class Observation(object):
         self.g1_field, self.g2_field, self.nshear = getShearFiled(config=self.config)
 
 
-    def runOneChip(self, chip, filt, chip_output, wcs_fp=None, psf_model=None, pointing_ID=0, ra_cen=None, dec_cen=None, img_rot=None, exptime=150., timestamp_obs=1621915200, pointing_type='MS', input_cat_name=None, shear_cat_file=None, cat_dir=None, sed_dir=None):
+    def runOneChip(self, chip, filt, chip_output, wcs_fp=None, psf_model=None, pointing_ID=0, ra_cen=None, dec_cen=None, img_rot=None, exptime=150., timestamp_obs=1621915200, pointing_type="MS", shear_cat_file=None, cat_dir=None, sed_dir=None):
 
         if (ra_cen is None) or (dec_cen is None):
-            ra_cen = self.config["ra_center"]
-            dec_cen = self.config["dec_center"]
+            ra_cen = self.config["obs_setting"]["ra_center"]
+            dec_cen = self.config["obs_setting"]["dec_center"]
         if img_rot is None:
-            img_rot = self.config["image_rot"]
+            img_rot = self.config["obs_setting"]["image_rot"]
 
-        if self.config["psf_model"] == "Gauss":
+        if self.config["psf_setting"]["psf_model"] == "Gauss":
             psf_model = PSFGauss(chip=chip)
-        elif self.config["psf_model"] == "Interp":
+        elif self.config["psf_setting"]["psf_model"] == "Interp":
             psf_model = PSFInterp(chip=chip)
         else:
             print("unrecognized PSF model type!!", flush=True)
@@ -77,12 +78,11 @@ class Observation(object):
         if chip.survey_type == "photometric":
             sky_map = None
         elif chip.survey_type == "spectroscopic":
-            skyfile = os.path.join(self.path_dict["data_dir"], 'skybackground/sky_emiss_hubble_50_50_A.dat')
-            sky_map = calculateSkyMap_split_g(xLen=chip.npix_x, yLen=chip.npix_y, blueLimit=filt.blue_limit, redLimit=filt.red_limit, skyfn=skyfile, conf=chip.sls_conf, pixelSize=chip.pix_scale, isAlongY=0)
+            sky_map = calculateSkyMap_split_g(xLen=chip.npix_x, yLen=chip.npix_y, blueLimit=filt.blue_limit, redLimit=filt.red_limit, skyfn=path_dict["sky_file"], conf=chip.sls_conf, pixelSize=chip.pix_scale, isAlongY=0)
 
         if pointing_type == 'MS':
             # Load catalogues and templates
-            self.cat = Catalog(config=self.config, chip=chip, cat_dir=cat_dir, sed_dir=sed_dir, pRa=ra_cen, pDec=dec_cen, rotation=img_rot, template_dir=self.path_dict["template_dir"])
+            self.cat = Catalog(config=self.config, chip=chip, cat_dir=cat_dir, sed_dir=sed_dir)
             self.nobj = len(self.cat.objs)
 
             # Loop over objects
@@ -97,7 +97,8 @@ class Observation(object):
                 # Load SED
                 if obj.type == 'star':
                     normF = chip.normF_star
-                    #continue
+                    if self.config["galaxy_only"]:
+                        continue
                     try:
                         obj.load_SED(
                             survey_type=chip.survey_type, 
@@ -109,18 +110,18 @@ class Observation(object):
                         continue
                 elif obj.type == 'galaxy': # or obj.type == quasar
                     normF = chip.normF_galaxy
-                    # continue
+                    if self.config["star_only"]:
+                        continue
                     obj.load_SED(
-                        sed_path=sed_dir, 
                         survey_type=chip.survey_type, 
                         sed_templates=self.cat.tempSed_gal, 
                         normFilter=normF, 
                         target_filt=filt)
                 elif obj.type == 'quasar':
                     normF = chip.normF_galaxy
-                    # continue
+                    if self.config["star_only"]:
+                        continue
                     obj.load_SED(
-                        sed_path=sed_dir, 
                         survey_type=chip.survey_type, 
                         sed_templates=self.cat.tempSed_gal, 
                         normFilter=normF, 
@@ -140,14 +141,18 @@ class Observation(object):
                     # print(obj.getMagFilter(filt))
                     continue
 
-                if self.config["shear_method"] == "constant":
+                if self.config["shear_setting"]["shear_type"] == "constant":
                     if obj.type == 'star':
                         g1, g2 = 0, 0
                     else:
                         g1, g2 = self.g1_field, self.g2_field
-                elif self.config["shear_method"] == "extra":
-                # TODO: every object with individual shear from input catalog(s)
-                    g1, g2 = self.g1_field[j], self.g2_field[j]
+                elif self.config["shear_setting"]["shear_type"] == "extra":
+                    try:
+                        # TODO: every object with individual shear from input catalog(s)
+                        g1, g2 = self.g1_field[j], self.g2_field[j]
+                    except:
+                        print("failed to load external shear.")
+                        pass
 
                 pos_img, offset, local_wcs = obj.getPosImg_Offset_WCS(img=chip.img, fdmodel=self.fd_model, chip=chip, verbose=False)
                 if pos_img.x == -1 or pos_img.y == -1:
@@ -159,7 +164,9 @@ class Observation(object):
 
                 # Draw object & update output catalog
                 try:
-                    if chip.survey_type == "photometric":
+                    if self.config["out_cat_only"]:
+                        isUpdated = True
+                    if chip.survey_type == "photometric" and not self.config["out_cat_only"]:
                         isUpdated, pos_shear = obj.drawObj_multiband(
                             tel=self.tel,
                             pos_img=pos_img, 
@@ -170,7 +177,7 @@ class Observation(object):
                             g1=g1, 
                             g2=g2, 
                             exptime=exptime)
-                    elif chip.survey_type == "spectroscopic":
+                    elif chip.survey_type == "spectroscopic" and not self.config["out_cat_only"]:
                         isUpdated, pos_shear = obj.drawObj_slitless(
                             tel=self.tel, 
                             pos_img=pos_img, 
@@ -203,8 +210,6 @@ class Observation(object):
 
         # Detector Effects
         # ===========================================================
-        # chip.img = chip.addNoise(config=self.config, tel=self.tel, filt=filt, img=chip.img, sky_map=sky_map)
-        # chip.img = chip.addEffects(config=self.config, img=chip.img, chip_output=chip_output, filt=filt, pointing_ID=pointing_ID)
 
         # whether to output zero, dark, flat calibration images.
         chip.img = chip.addEffects(
@@ -253,6 +258,7 @@ class Observation(object):
                 col_num=chip.colID,
                 extName='raw')
             chip.img = galsim.Image(chip.img.array, dtype=np.uint16)
+            # chip.img = galsim.Image(chip.img.array, dtype=np.uint32)
             hdu1 = fits.PrimaryHDU(header=h_prim)
             hdu2 = fits.ImageHDU(chip.img.array, header=h_ext)
             hdu1 = fits.HDUList([hdu1, hdu2])
@@ -266,13 +272,13 @@ class Observation(object):
         print("check running:2: pointing-{:} chip-{:} pid-{:} memory-{:6.2}GB".format(pointing_ID, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) ), flush=True)
 
 
-    def runExposure(self, ra_cen=None, dec_cen=None, timestamp_obs=1621915200, pointing_ID=0, pointing_type='MS', img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None, oneChip=None):
+    def runExposure(self, ra_cen=None, dec_cen=None, timestamp_obs=1621915200, pointing_ID=0, pointing_type='MS', img_rot=None, exptime=150., shear_cat_file=None, chips=None):
 
         if (ra_cen == None) or (dec_cen == None):
-            ra_cen = self.config["ra_center"]
-            dec_cen = self.config["dec_center"]
+            ra_cen = self.config["obs_setting"]["ra_center"]
+            dec_cen = self.config["obs_setting"]["dec_center"]
         if img_rot == None:
-            img_rot = self.config["image_rot"]
+            img_rot = self.config["obs_setting"]["image_rot"]
 
         sub_img_dir, prefix = makeSubDir_PointingList(path_dict=self.path_dict, config=self.config, pointing_ID=pointing_ID)
 
@@ -281,9 +287,9 @@ class Observation(object):
             chip = self.chip_list[i]
             filt = self.filter_list[i]
 
-            # Just run one chip
-            if oneChip is not None:
-                if chip.chipID != oneChip:
+            # Only run a particular set of chips
+            if chips is not None:
+                if chip.chipID not in chips:
                     continue
 
             # Prepare output files
@@ -309,16 +315,30 @@ class Observation(object):
                 exptime=exptime, 
                 timestamp_obs=timestamp_obs,
                 pointing_type=pointing_type,
-                cat_dir=self.path_dict["cat_dir"], 
-                sed_dir=self.path_dict["SED_dir"])
+                cat_dir=self.path_dict["cat_dir"])
             print("finished running chip#%d..."%(chip.chipID), flush=True)
 
-    def runExposure_MPI_PointingList(self, ra_cen=None, dec_cen=None, pRange=None, timestamp_obs=np.array([1621915200]), pointing_type=np.array(['MS']), img_rot=None, exptime=150., input_cat_name=None, shear_cat_file=None):
-        comm = MPI.COMM_WORLD
-        ind_thread = comm.Get_rank()
-        num_thread = comm.Get_size()
+    def runExposure_MPI_PointingList(self, ra_cen=None, dec_cen=None, pRange=None, timestamp_obs=np.array([1621915200]), pointing_type=np.array(['MS']), img_rot=None, exptime=150., shear_cat_file=None, chips=None, use_mpi=False):
+        if use_mpi:
+            comm = MPI.COMM_WORLD
+            ind_thread = comm.Get_rank()
+            num_thread = comm.Get_size()
 
-        nchips_per_fp = len(self.chip_list)
+        if chips is None:
+            nchips_per_fp = len(self.chip_list)
+            run_chips = self.chip_list
+            run_filts = self.filter_list
+        else:
+            # Only run a particular set of chips
+            run_chips = []
+            run_filts = []
+            nchips_per_fp = len(chips)
+            for ichip in range(len(self.chip_list)):
+                chip = self.chip_list[ichip]
+                filt = self.filter_list[ichip]
+                if chip.chipID in chips:
+                    run_chips.append(chip)
+                    run_filts.append(filt)
 
         # TEMP
         if len(timestamp_obs) == 1:
@@ -343,15 +363,18 @@ class Observation(object):
                     pointing_ID = pStart + ipoint
                 else:
                     pointing_ID = pRange[ipoint]
-                if i % num_thread != ind_thread:
-                    continue
+                if use_mpi:
+                    if i % num_thread != ind_thread:
+                        continue
 
                 pid = os.getpid()
 
                 sub_img_dir, prefix = makeSubDir_PointingList(path_dict=self.path_dict, config=self.config, pointing_ID=pointing_ID)
 
-                chip = self.chip_list[ichip]
-                filt = self.filter_list[ichip]
+                # chip = self.chip_list[ichip]
+                # filt = self.filter_list[ichip]
+                chip = run_chips[ichip]
+                filt = run_filts[ichip]
                 print("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid), flush=True)
                 chip_output = ChipOutput(
                     config=self.config, 
@@ -374,6 +397,5 @@ class Observation(object):
                     exptime=exptime,
                     timestamp_obs=timestamp_obs[ipoint],
                     pointing_type=pointing_type[ipoint],
-                    cat_dir=self.path_dict["cat_dir"], 
-                    sed_dir=self.path_dict["SED_dir"])
+                    cat_dir=self.path_dict["cat_dir"])
                 print("finished running chip#%d..."%(chip.chipID), flush=True)

ObservationSim/PSF/PSFInterp/PSFConfig.py

@@ -3,7 +3,8 @@ import numpy as np
 import scipy.io
 from itertools import islice
 
-from PSF.PSFInterp.PSFUtil import *
+# from PSF.PSFInterp.PSFUtil import *
+# from PSFUtil import findNeighbors, findNeighbors_hoclist
 
 
 
@@ -141,6 +142,8 @@ def psfMaker_IDW(px, py, PSFMat, cen_col, cen_row, IDWindex=2, OnlyNeighbors=Tru
     npsf = PSFMat[:, :, :].shape[0]
     psfWeight = np.zeros([npsf])
 
+    from .PSFUtil import findNeighbors, findNeighbors_hoclist
+
     if OnlyNeighbors == True:
         if hoc is None:
             neigh = findNeighbors(px, py, cen_col, cen_row, dr=5., dn=4, OnlyDistance=False)

ObservationSim/PSF/PSFInterp/PSFInterp.py

@@ -10,9 +10,10 @@ import numpy as np
 import os
 import time
 import copy
-import PSF.PSFInterp.PSFConfig as myConfig
-import PSF.PSFInterp.PSFUtil as myUtil
-from PSF.PSFModel import PSFModel
+
+from . import PSFConfig as myConfig
+from . import PSFUtil as myUtil
+from ..PSFModel import PSFModel
 
 LOG_DEBUG = False  #***#
 NPSF      = 900    #***# 30*30
@@ -29,6 +30,7 @@ class PSFInterp(PSFModel):
             params: Other parameters?
             PSF_data_file: The file for PSF data matrix (optional).
         """
+        from . import PSFUtil as myUtil
         if LOG_DEBUG:
             print('===================================================')
             print('DEBUG: psf module for csstSim ' \
@@ -55,6 +57,7 @@ class PSFInterp(PSFModel):
         self.cen_row= np.zeros([self.nwave, NPSF], dtype=np.float32)
         self.hoc    =[]
         self.hoclist=[]
+        
         for twave in range(self.nwave):
             for tpsf in range(NPSF):
                 self.psfMat[twave, tpsf, :, :] = self.PSF_data[twave][tpsf]['psfMat']
@@ -100,6 +103,7 @@ class PSFInterp(PSFModel):
         Returns:
             psfSet: The matrix of the csst-psf
         """
+        from . import PSFConfig as myConfig
         psfSet = []
         for ii in range(self.nwave):
             iwave = ii+1
@@ -144,7 +148,7 @@ class PSFInterp(PSFModel):
         Returns:
             PSF: A 'galsim.GSObject'.
         """
-
+        from . import PSFConfig as myConfig
         pixSize = np.rad2deg(self.pixsize*1e-3/28)*3600  #set psf pixsize
 
         #***#  assert self.iccd == chip.chipID, 'ERROR: self.iccd != chip.chipID'
@@ -200,8 +204,6 @@ class PSFInterp(PSFModel):
         return self.psf.shear(PSFshear), PSFshear
 
 
-
-
 if __name__ == '__main__':
     if True:
         psfPath = '/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/CSSOS_psf_20210108/CSST_psf_ciomp_2p5um_cycle3_ccr90'
@@ -256,6 +258,8 @@ if __name__ == '__main__':
         from scipy import ndimage
         y, x = ndimage.center_of_mass(img)
         plt.plot([x-dng],[y-dng], 'rx', ms = 10, mew=4)
+
+        from . import PSFUtil as myUtil
         x, y = myUtil.findMaxPix(img)
         plt.plot([x-dng],[y-dng], 'b+', ms = 10, mew=4)
         plt.savefig('test/figs/test2.jpg')

ObservationSim/PSF/PSFInterp/PSFProcess.py

@@ -6,8 +6,10 @@ import mpi4py.MPI as MPI
 
 
 #sys.path.append("/public/home/weichengliang/lnData/CSST_new_framwork/csstPSF_20201222")
-import PSFConfig as myConfig
-import PSFUtil as myUtil
+# import PSFConfig as myConfig
+# import PSFUtil as myUtil
+import PSFConfig
+import PSFUtil
 
 def mkdir(path):
     isExists = os.path.exists(path)
@@ -55,8 +57,7 @@ for iccd in range(1, 31):
             #    continue
 
             ipsfOutput = psfMatPath + '/psf_{:}_centroidWgt.mat'.format(ipsf)
-
-            psfInfo = myConfig.LoadPSF(iccd, iwave, ipsf, psfPath, InputMaxPixelPos=True)
+            psfInfo = PSFConfig.LoadPSF(iccd, iwave, ipsf, psfPath, InputMaxPixelPos=True)
             ipsfMat  = psfInfo['psfMat']
             npix_y, npix_x = ipsfMat.shape
 
@@ -67,7 +68,7 @@ for iccd in range(1, 31):
             ncut = int(npix_y*0.90)
             ncut = ncut + ncut%2  #even pixs
 
-            img, cx, cy = myUtil.centroidWgt(ipsfMat, nt=ncut)
+            img, cx, cy = PSFUtil.centroidWgt(ipsfMat, nt=ncut)
             dcx = cx - npix_x/2   #pixel coords -> global coords
             dcy = cy - npix_y/2  #pixel coords -> global coords
             dcx*= psfInfo['pixsize']*1e-3 #5e-3  #pixels -> mm