Baseline

ObservationSim/_util.py

+import numpy as np
+import os
+from datetime import datetime
+
+def imgName(tt=0):
+	ut = datetime.utcnow()
+	eye, emo, eda, eho, emi, ese = str(ut.year), str(ut.month), str(ut.day), str(ut.hour), str(ut.minute), str(ut.second)
+	emse = str(ut.microsecond)
+	if int(emo)<10: emo = "0%s"%emo
+	if int(eda)<10: eda = "0%s"%eda
+	if int(eho)<10: eho = "0%s"%eho
+	if int(emi)<10: emi = "0%s"%emi
+	if int(ese)<10: ese = "0%s"%ese
+
+	if tt==0:
+		namekey = "CSST%s%s%sT%s%s%s"%(eye,emo,eda,eho,emi,ese)
+	elif tt==1:
+		namekey = "%s-%s-%sT%s:%s:%s.%s"%(eye,emo,eda,eho,emi,ese,emse)
+	elif tt==2:
+		namekey = "%s%s%s%s%s%s"%(eye,emo,eda,eho,emi,ese)
+	else:
+		raise ValueError("!!! Give a right 'tt' value.")
+
+	return namekey
+
+def makeSubDir(path_dict, config):
+	subImgdir = path_dict["output_img_dir"] + config["mockImgDir"] + '/'
+	if not os.path.exists(subImgdir):
+		os.system("mkdir %s"%subImgdir)
+	imgKey0 = imgName(tt=0)
+	imgKey1 = imgName(tt=1)
+	imgKey2 = imgName(tt=2)
+	subImgdir = subImgdir + imgKey0 + config["reIndex"] + config["reShear"] + "/"
+	if not os.path.exists(subImgdir):
+		os.system("mkdir %s"%subImgdir)
+	return subImgdir, imgKey0, imgKey1, imgKey2
+
+def makeSubDir_PointingList(path_dict, config, pointing_ID=0):
+	imgDir = path_dict["work_dir"] + config["mockImgDir"] + '/'
+	if not os.path.exists(imgDir):
+		os.system("mkdir %s"%imgDir)
+	# prefix = "MSC_" + config["date_obs"] + config["time_obs"] + "_" + str(pointing_ID).rjust(7, '0')
+	prefix = "MSC_" + str(pointing_ID).rjust(7, '0')
+	subImgdir = os.path.join(imgDir, prefix)
+	if not os.path.exists(subImgdir):
+		os.system("mkdir %s"%subImgdir)
+	return subImgdir, prefix
+
+def getShearFiled(config, shear_cat_file=None):
+	if not config["shear_method"] in ["constant", "extra"]:
+		raise ValueError("Please set a right 'shear_method' parameter.")
+
+	if config["shear_method"] == "constant":
+		g1 = config["reduced_g1"]
+		g2 = config["reduced_g2"]
+		reduced_shear = np.sqrt(g1**2 + g2**2)
+		nshear = 1
+		# TODO logging
+	else:
+		# TODO logging
+		# if not os.path.exists(shear_cat_file):
+		# 	raise ValueError("Cannot find shear catalog file.")
+		# shearCat = np.loadtxt(shear_cat_file)
+		# nshear = shearCat.shape[0]
+		# g1, g2 = shearCat[:, 0], shearCat[:, 1]
+		pass
+	return g1, g2, nshear
\ No newline at end of file

ObservationSim/mpd.hosts

+comput101
+comput102
+comput103
+comput104
+comput105
+comput106
+comput107
+comput108
+comput109
+comput110
+comput111
+comput112
+comput113
+smp1
+admin
+

ObservationSim/preprocess.py

+import os
+import numpy as np
+from Pointing import *
+
+# Make directories
+imgDir = os.path.join(work_dir, config["mockImgDir"])
+if not os.path.exists(imgDir):
+    os.system("mkdir %s"%imgDir)
+
+# prefix = "MSC_" + config["date_obs"] + "_" + config["time_obs"] + "_"
+prefix = "MSC_"
+
+for pointing_ID in range(len(pRA)):
+    if pointing_ID not in pRange:
+        continue
+    fname = prefix + str(pointing_ID).rjust(7, '0')
+    subImgDir = os.path.join(imgDir, fname)
+    if not os.path.exists(subImgDir):
+        os.system("mkdir %s"%subImgDir)
\ No newline at end of file

ObservationSim/run.pbs

+#!/bin/bash
+
+#PBS -N SIMS
+##PBS -l walltime=70:00:00
+
+##mpdallexit
+##mpdboot -n 10 -f  ./mpd.hosts
+
+##PBS -j oe
+#PBS -l nodes=comput110:ppn=80
+#####PBS -q longq
+#PBS -q batch
+#PBS -u fangyuedong
+
+NP=60
+date
+echo $NP
+
+# mpirun -np $NP --oversubscribe -H comput101 python /public/home/fangyuedong/CSST_new_framework/ObservationSim/runExposure.py
+
+python /public/home/fangyuedong/CSST_new_framework/ObservationSim/preprocess.py
+mpirun -np $NP python /public/home/fangyuedong/CSST_new_framework/ObservationSim/runExposure.py

ObservationSim/runExposure.py

+from ObservationSim import Observation
+from Pointing import *
+import os
+import numpy as np
+import galsim
+
+import gc
+gc.enable()
+
+#############################################
+# Testing run one exposure (NOT using MPI)
+ipoint = 2
+pointRa = float('{:8.4f}'.format(pRA[ipoint]))
+pointDec = float('{:8.4f}'.format(pDEC[ipoint]))
+
+cat_dir = 'point_RA' + str(pointRa) + '_DE' + str(pointDec) + '/'
+obs = Observation(input_cat_dir=cat_dir, work_dir=work_dir, data_dir=data_dir)
+obs = Observation(work_dir=work_dir, data_dir=data_dir)
+obs.runExposure(ra_cen=pointRa, dec_cen=pointDec, pointing_ID=ipoint, oneChip=16)
+
+#############################################
+# Testing run pointing list (using MPI)
+# obs = Observation(work_dir=work_dir, data_dir=data_dir)
+# obs.runExposure_MPI_PointingList(ra_cen=pRA, dec_cen=pDEC, pRange=pRange)

test/ObservationSim.cfg

+##################################################
+#  Default configuration for CSSOS simulation    #
+#     CSST-Sim Group, 2021/01/04, version 0.2    #
+#                                                #
+##################################################
+
+# Observation setting
+date_obs      210304                  # Observation date [yymmdd]
+time_obs      093000                  # Observation time [hhmmss]
+ra_center     60.0                    # Telesscope pointing center [degree] (Default)
+dec_center   -40.0                     
+psf_rcont     0.15,0.8                # Radius of the 80% flux concentration (for a Gaussian PSF)
+image_rot     23.4333                 # Image orientation [degree]
+sigma_spin    0.0                     # Image spin [degree]
+rotateEll     0.0                     # Rotate the intrinsic ellipticity by a given angle  
+                                      # (currently only support 0.0, 45.0, 90.0, 135.0 [degree])
+survey_type   Photometric             # Survey simulation option:
+                                      # 'Photometric': only run sims for photometric chips
+                                      # 'Spectroscopic': only run sims for spectroscopic chps
+                                      # 'All': run sims for all chips
+psf_model     Interp                  # which PSF model to use:
+                                      # 'Gauss': simple gaussin profile
+                                      # 'Interp': Interpolated PSF from measured data 
+
+#===========================================================
+# Instrument effects
+#===========================================================
+field_dist       y                    # Y/N: Whether to add field distortion
+abs_back         y                    # Y: add sky background + sky noise & dark + dark noise 
+                                      # N: only add sky noise and dark noise
+add_bias         y                    # Y/N: Whether to add bias-level to image
+shutter_effect   y                    # Y/N: Whether to apply shutter effect
+shutter_output   y                    # Y/N: Whether to export shutter effect 16-bit image (<=65535)
+flat_fielding    y                    # Y/N: Whether to add flat-fielding effect
+bias_output      y                    # Y/N: Whether to export bias frames
+dark_output      y                    # Y/N: Whether to export the combined dark current file
+flat_output      y                    # Y/N: Whether to export the combined flat-field file
+prnu_effect      y                    # Y/N: Whether to apply PRNU effect
+prnu_output      y                    # Y/N: Whether to export the PRNU (pixel-to-pixel flat-fielding) file
+non_linear       y                    # Y/N: Whether to apply non-linearity
+cosmic_ray       y                    # Y/N: Whether to add cosmic-ray
+cte_trail        y                    # Y/N: Whether to simulate CTE trails
+saturbloom       y                    # Y/N: Whether to simulate Saturation and Blooming
+add_badcolumns   y                    # Y/N: Whether to add bad columns
+add_hotpixels    y                    # Y/N: Whether to add hot pixels
+add_deadpixels   y                    # Y/N: Whether to add dead(dark) pixels
+readout16        n                    # Y/N: Whether to export as 16 subimages (channels) with pre- and over-scan
+bright_fatter    y                    # Y/N: Whether to add brighter-fatter (also electron diffusion) effects
+df_strength      2.3                  # Sillicon sensor diffusion strength
+bias_level       500                  # e-/pixel
+gain             1.1                  # Gain
+full_well        90000                # Full well level [e-]
+NBias            10                   # number of frames to be exported
+NDark            10                   # number of frames to be exported
+NFlat            10                   # number of frames to be exported
+
+#===========================================================
+# Shear method
+#===========================================================
+shear_method  constant                # Only two methods are provided to generate mock shear field:
+                                      # 'constant': all galaxies are assigned a constant reduced shear,
+                                      #             but with different betas (orientations)
+                                      # 'extra'   : from catalog(s) (not available yet)
+
+# Constant shear field
+reduced_g1  0.026                     # g1 component when 'shear_method'='constant'
+reduced_g2  0.015                     # g2 component when 'shear_method'='constant'
+reShear     E                         # Representation of the shear vertor
+
+# Extra shear file
+# shear_cat   mockShear.cat           # Extra provided shear catalog (currently not used)
+
+#===========================================================
+# Output directories
+#===========================================================
+mockImgDir    TEST                    # Name of subdirectory to save the mock image
+
+#===========================================================
+# Random Seeds
+#===========================================================
+seed_mock            12345678         # Seed for generating random values of ra, dec and av
+seed_star            121212121        # Seed for generating random redshift, sed_type for stars
+seed_gal             212121212        # Seed for generating random redshift, sed_type for galaxies
+seed_Av              121212           # Seed for generating random intrinsic extinction
+seed_CR              20210317         # Seed for generating random cosmic ray map
+seed_flat            20210101         # Seed for generating random flat field
+seed_prnu            20210102         # Seed for photo-response non-uniformity
+seed_skynoise        20210201         # Seed for sky noise
+seed_gainNonUniform  20210202         # Seed for Gain nonuniformity
+seed_biasNonUniform  20210203         # Seed for Bias nonuniformity
+seed_rnNonUniform    20210204         # Seed for ReadNoise nonuniformity
+seed_badcolumns      20240309         # Initial Seed for Bad Columns
+seed_defective       20210304         # Seed for Defective(bad) pixels
\ No newline at end of file