Merge remote-tracking branch 'origin/sim_scheduler' into develop (f5c20f83) · Commits · csst-sims / csst_msc_sim

.gitignore

+3 −1

Original line number	Diff line number	Diff line
		@@ -6,3 +6,5 @@ dist/*
		*disperse.c
		*interp.c
		!*libshao.so
		*.out
		pnodes
		No newline at end of file

Catalog/C6_50sqdeg_ns.py

+3 −4

Original line number	Diff line number	Diff line
		@@ -39,9 +39,9 @@ bundle_file_list = ['galaxies_C6_bundle000199.h5','galaxies_C6_bundle000200.h5',
		qsosed_file_list = ['quickspeclib_interp1d_run1.fits','quickspeclib_interp1d_run2.fits','quickspeclib_interp1d_run3.fits','quickspeclib_interp1d_run4.fits','quickspeclib_interp1d_run5.fits','quickspeclib_interp1d_run6.fits','quickspeclib_interp1d_run7.fits','quickspeclib_interp1d_run8.fits','quickspeclib_interp1d_run9.fits','quickspeclib_interp1d_run10.fits','quickspeclib_interp1d_run11.fits','quickspeclib_interp1d_run12.fits','quickspeclib_interp1d_run13.fits','quickspeclib_interp1d_run14.fits','quickspeclib_interp1d_run15.fits','quickspeclib_interp1d_run16.fits','quickspeclib_interp1d_run17.fits','quickspeclib_interp1d_run18.fits','quickspeclib_interp1d_run19.fits','quickspeclib_interp1d_run20.fits','quickspeclib_interp1d_run21.fits','quickspeclib_interp1d_run22.fits','quickspeclib_interp1d_run23.fits','quickspeclib_interp1d_run24.fits','quickspeclib_interp1d_run25.fits','quickspeclib_interp1d_run26.fits','quickspeclib_interp1d_run27.fits','quickspeclib_interp1d_run28.fits','quickspeclib_interp1d_run29.fits','quickspeclib_interp1d_run30.fits']

		# star_file_list = ['C7_Gaia_Galaxia_RA170DECm23_healpix.hdf5', 'C7_Gaia_Galaxia_RA180DECp60_healpix.hdf5', 'C7_Gaia_Galaxia_RA240DECp30_healpix.hdf5', 'C7_Gaia_Galaxia_RA300DECm60_healpix.hdf5', 'C7_Gaia_Galaxia_RA30DECm48_healpix.hdf5']
		star_center_list = [(170., -23.), (180., 60.), (240., 30.), (300., -60.), (30., -48.)]
		star_center_list = [(170., -23.), (180., 60.), (240., 30.), (300., -60.), (30., -48.),[246.5, 40]]

		star_file_list = ['C9_RA170_DECm23_calmag_Nside_128_healpix.hdf5', 'C9_RA180_DECp60_calmag_Nside_128_healpix.hdf5', 'C9_RA240_DECp30_calmag_Nside_128_healpix.hdf5', 'C9_RA300_DECm60_calmag_Nside_128_healpix.hdf5', 'C9_RA30_DECm48_calmag_Nside_128_healpix.hdf5']
		star_file_list = ['C9_RA170_DECm23_calmag_Nside_128_healpix.hdf5', 'C9_RA180_DECp60_calmag_Nside_128_healpix.hdf5', 'C9_RA240_DECp30_calmag_Nside_128_healpix.hdf5', 'C9_RA300_DECm60_calmag_Nside_128_healpix.hdf5', 'C9_RA30_DECm48_calmag_Nside_128_healpix.hdf5','trilegal_calMag_mpi_Nside_128_healpix.hdf5']

		class StarParm(ctypes.Structure):
		_fields_ = [
		@@ -187,7 +187,7 @@ class Catalog(CatalogBase):
		rv_c = obj.param['rv']/(atcons.c.value/1000.)
		Doppler_factor = np.sqrt((1+rv_c)/(1-rv_c))
		wave_RV = wave*Doppler_factor
		return wave_RV, spec
		return wave_RV, np.power(10,spec[:])

		def _load_SED_lib_gals(self):
		pcs = h5.File(os.path.join(self.galaxy_SED_path, "pcs.h5"), "r")
		@@ -511,7 +511,6 @@ class Catalog(CatalogBase):
		# erg/s/cm2/A --> photon/s/m2/A
		all_sed = y * lamb / (cons.h.value * cons.c.value) * 1e-13
		sed = Table(np.array([lamb, all_sed]).T, names=('WAVELENGTH', 'FLUX'))

		if obj.type == 'quasar':
		# integrate to get the magnitudes
		sed_photon = np.array([sed['WAVELENGTH'], sed['FLUX']]).T

ObservationSim/Config/ChipOutput.py

+31 −24

Original line number	Diff line number	Diff line
		@@ -3,8 +3,9 @@ import logging
		import ObservationSim.Config._util as _util
		from ObservationSim.Config.Header import generatePrimaryHeader


		class ChipOutput(object):
		def __init__(self, config, chip, filt, pointing):
		def __init__(self, config, chip, filt, pointing, logger_filename=None):
		self.config = config
		self.chip = chip
		self.filt = filt
		@@ -29,18 +30,22 @@ class ChipOutput(object):
		run_counter=self.config["run_counter"],
		chip_name=self.chip_label)

		obs_id = _util.get_obs_id(img_type=self.pointing_type, project_cycle=config["project_cycle"], run_counter=config["run_counter"], pointing_id=pointing.obs_id, pointing_type_code = pointing.pointing_type_code)
		obs_id = _util.get_obs_id(img_type=self.pointing_type, project_cycle=config["project_cycle"], run_counter=config[
		"run_counter"], pointing_id=pointing.obs_id, pointing_type_code=pointing.pointing_type_code)

		self.subdir = pointing.output_dir
		self.cat_name = self.h_prim['FILENAME'] + '.cat'
		if logger_filename is None:
		logger_filename = self.h_prim['FILENAME'] + '.log'

		self.logger = logging.getLogger()
		fh = logging.FileHandler(os.path.join(self.subdir, logger_filename), mode='w+', encoding='utf-8')
		fh = logging.FileHandler(os.path.join(
		self.subdir, logger_filename), mode='w+', encoding='utf-8')
		fh.setLevel(logging.DEBUG)
		self.logger.setLevel(logging.DEBUG)
		logging.getLogger('numba').setLevel(logging.WARNING)
		formatter = logging.Formatter('%(asctime)s - %(msecs)d - %(levelname)-8s - [%(filename)s:%(lineno)d] - %(message)s')
		formatter = logging.Formatter(
		'%(asctime)s - %(msecs)d - %(levelname)-8s - [%(filename)s:%(lineno)d] - %(message)s')
		fh.setFormatter(formatter)
		self.logger.addHandler(fh)

		@@ -67,7 +72,8 @@ class ChipOutput(object):
		def create_output_file(self):
		if self.pointing_type == 'SCI':
		self.cat = open(os.path.join(self.subdir, self.cat_name), "w")
		self.logger.info("Creating catalog file %s ...\n"%(os.path.join(self.subdir, self.cat_name)))
		self.logger.info("Creating catalog file %s ...\n" %
		(os.path.join(self.subdir, self.cat_name)))
		if not self.hdr.endswith("\n"):
		self.hdr += "\n"
		self.cat.write(self.hdr)
		@@ -77,7 +83,8 @@ class ChipOutput(object):
		yimg = obj.real_pos.y + 1.0

		line = self.fmt % (
		obj.id, int(self.chip_label), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.ra_orig, obj.dec_orig, obj.z, obj.getMagFilter(self.filt), obj.type,
		obj.id, int(self.chip_label), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.ra_orig, obj.dec_orig, obj.z, obj.getMagFilter(
		self.filt), obj.type,
		obj.pmra, obj.pmdec, obj.rv, obj.parallax)
		line += obj.additional_output_str
		if not line.endswith("\n"):

ObservationSim/MockObject/CatalogBase.py

+62 −57

Original line number	Diff line number	Diff line
		@@ -87,27 +87,32 @@ class CatalogBase(metaclass=ABCMeta):
		return e1, e2, e_total

		@staticmethod
		def convert_sed(mag, sed, target_filt, norm_filt=None):
		def convert_sed(mag, sed, target_filt, norm_filt=None, mu=1.):
		bandpass = target_filt.bandpass_full

		if norm_filt is not None:
		norm_thr_rang_ids = norm_filt['SENSITIVITY'] > 0.001
		else:
		norm_filt = Table(
		np.array(np.array([bandpass.wave_list*10.0, bandpass.func(bandpass.wave_list)])).T, names=(['WAVELENGTH', 'SENSITIVITY'])
		np.array(np.array([bandpass.wave_list*10.0, bandpass.func(
		bandpass.wave_list)])).T, names=(['WAVELENGTH', 'SENSITIVITY'])
		)
		norm_thr_rang_ids = norm_filt['SENSITIVITY'] > 0.001

		sedNormFactor = getNormFactorForSpecWithABMAG(ABMag=mag,
		spectrum=sed,
		norm_thr=norm_filt,
		sWave=np.floor(norm_filt[norm_thr_rang_ids][0][0]),
		sWave=np.floor(
		norm_filt[norm_thr_rang_ids][0][0]),
		eWave=np.ceil(norm_filt[norm_thr_rang_ids][-1][0]))
		sed_photon = copy.copy(sed)
		sed_photon = np.array([sed_photon['WAVELENGTH'], sed_photon['FLUX']*sedNormFactor]).T
		sed_photon = galsim.LookupTable(x=np.array(sed_photon[:, 0]), f=np.array(sed_photon[:, 1]), interpolant='nearest')
		sed_photon = np.array(
		[sed_photon['WAVELENGTH'], sed_photon['FLUX']*sedNormFactor]).T
		sed_photon = galsim.LookupTable(x=np.array(sed_photon[:, 0]), f=np.array(
		sed_photon[:, 1] * mu), interpolant='nearest')
		# Get magnitude
		sed_photon = galsim.SED(sed_photon, wave_type='A', flux_type='1', fast=False)
		sed_photon = galsim.SED(sed_photon, wave_type='A',
		flux_type='1', fast=False)
		interFlux = integrate_sed_bandpass(sed=sed_photon, bandpass=bandpass)
		mag_csst = getABMAG(
		interFlux=interFlux,

ObservationSim/MockObject/Galaxy.py

+84 −54

Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@ from ObservationSim.MockObject.MockObject import MockObject

		# import tracemalloc


		class Galaxy(MockObject):
		def __init__(self, param, logger=None):
		super().__init__(param, logger=logger)
		@@ -16,6 +17,11 @@ class Galaxy(MockObject):
		self.disk_sersic_idx = 1.
		if not hasattr(self, "bulge_sersic_idx"):
		self.bulge_sersic_idx = 4.
		if not hasattr(self, "mu"):
		if hasattr(self, "detA"):
		self.mu = 1./self.detA
		else:
		self.mu = 1.

		def unload_SED(self):
		"""(Test) free up SED memory
		@@ -24,14 +30,16 @@ class Galaxy(MockObject):

		def getGSObj_multiband(self, tel, psf_list, bandpass_list, filt, nphotons_tot=None, g1=0, g2=0, exptime=150., fd_shear=None):
		if len(psf_list) != len(bandpass_list):
		raise ValueError("!!!The number of PSF profiles and the number of bandpasses must be equal.")
		raise ValueError(
		"!!!The number of PSF profiles and the number of bandpasses must be equal.")
		objs = []
		if nphotons_tot == None:
		nphotons_tot = self.getElectronFluxFilt(filt, tel, exptime)
		# print("nphotons_tot = ", nphotons_tot)

		try:
		full = integrate_sed_bandpass(sed=self.sed, bandpass=filt.bandpass_full)
		full = integrate_sed_bandpass(
		sed=self.sed, bandpass=filt.bandpass_full)
		except Exception as e:
		print(e)
		if self.logger:
		@@ -54,10 +62,12 @@ class Galaxy(MockObject):
		return -1

		psf = psf_list[i]
		disk = galsim.Sersic(n=self.disk_sersic_idx, half_light_radius=self.hlr_disk, flux=1.0)
		disk = galsim.Sersic(n=self.disk_sersic_idx,
		half_light_radius=self.hlr_disk, flux=1.0)
		disk_shape = galsim.Shear(g1=self.e1_disk, g2=self.e2_disk)
		disk = disk.shear(disk_shape)
		bulge = galsim.Sersic(n=self.bulge_sersic_idx, half_light_radius=self.hlr_bulge, flux=1.0)
		bulge = galsim.Sersic(n=self.bulge_sersic_idx,
		half_light_radius=self.hlr_bulge, flux=1.0)
		bulge_shape = galsim.Shear(g1=self.e1_bulge, g2=self.e2_bulge)
		bulge = bulge.shear(bulge_shape)

		@@ -67,13 +77,15 @@ class Galaxy(MockObject):
		gal = bulge
		else:
		gal = self.bfrac * bulge + (1.0 - self.bfrac) * disk
		gal = gal.withFlux(nphotons)
		if fd_shear is not None:
		g1 += fd_shear.g1
		g2 += fd_shear.g2
		gal_shear = galsim.Shear(g1=g1, g2=g2)
		gal = gal.shear(gal_shear)
		# Magnification
		gal = gal.magnify(self.mu)
		gal = galsim.Convolve(psf, gal)
		gal = gal.withFlux(nphotons)

		objs.append(gal)
		final = galsim.Sum(objs)
		@@ -85,7 +97,8 @@ class Galaxy(MockObject):
		# print("nphotons_tot = ", nphotons_tot)

		try:
		full = integrate_sed_bandpass(sed=self.sed, bandpass=filt.bandpass_full)
		full = integrate_sed_bandpass(
		sed=self.sed, bandpass=filt.bandpass_full)
		except Exception as e:
		print(e)
		if self.logger:
		@@ -121,10 +134,12 @@ class Galaxy(MockObject):
		is_updated = 0

		# Model the galaxy as disk + bulge
		disk = galsim.Sersic(n=self.disk_sersic_idx, half_light_radius=self.hlr_disk, flux=1.0, gsparams=gsp)
		disk = galsim.Sersic(
		n=self.disk_sersic_idx, half_light_radius=self.hlr_disk, flux=1.0, gsparams=gsp)
		disk_shape = galsim.Shear(g1=self.e1_disk, g2=self.e2_disk)
		disk = disk.shear(disk_shape)
		bulge = galsim.Sersic(n=self.bulge_sersic_idx, half_light_radius=self.hlr_bulge, flux=1.0, gsparams=gsp)
		bulge = galsim.Sersic(
		n=self.bulge_sersic_idx, half_light_radius=self.hlr_bulge, flux=1.0, gsparams=gsp)
		bulge_shape = galsim.Shear(g1=self.e1_bulge, g2=self.e2_bulge)
		bulge = bulge.shear(bulge_shape)

		@@ -155,7 +170,8 @@ class Galaxy(MockObject):
		# print("nphotons_sub-band_%d = %.2f"%(i, nphotons))

		# Get PSF model
		psf, pos_shear = psf_model.get_PSF(chip=chip, pos_img=pos_img, bandpass=bandpass, folding_threshold=folding_threshold)
		psf, pos_shear = psf_model.get_PSF(
		chip=chip, pos_img=pos_img, bandpass=bandpass, folding_threshold=folding_threshold)

		if self.bfrac == 0:
		gal_temp = disk
		@@ -164,10 +180,13 @@ class Galaxy(MockObject):
		else:
		gal_temp = self.bfrac * bulge + (1.0 - self.bfrac) * disk
		gal_temp = gal_temp.shear(gal_shear)
		gal_temp = gal_temp.withFlux(nphotons)
		# Magnification
		gal_temp = gal_temp.magnify(self.mu)
		if not big_galaxy: # Not apply PSF for very big galaxy
		gal_temp = galsim.Convolve(psf, gal_temp)

		gal_temp = gal_temp.withFlux(nphotons)

		if i == 0:
		gal = gal_temp
		else:
		@@ -184,7 +203,8 @@ class Galaxy(MockObject):
		# ERROR happens
		return 2, pos_shear
		stamp.setCenter(x_nominal, y_nominal)
		bounds = stamp.bounds & galsim.BoundsI(0, chip.npix_x - 1, 0, chip.npix_y - 1)
		bounds = stamp.bounds & galsim.BoundsI(
		0, chip.npix_x - 1, 0, chip.npix_y - 1)
		if bounds.area() > 0:
		chip.img.setOrigin(0, 0)
		chip.img[bounds] += stamp[bounds]
		@@ -209,7 +229,8 @@ class Galaxy(MockObject):
		norm_thr_rang_ids = normFilter['SENSITIVITY'] > 0.001
		sedNormFactor = getNormFactorForSpecWithABMAG(ABMag=self.param['mag_use_normal'], spectrum=self.sed,
		norm_thr=normFilter,
		sWave=np.floor(normFilter[norm_thr_rang_ids][0][0]),
		sWave=np.floor(
		normFilter[norm_thr_rang_ids][0][0]),
		eWave=np.ceil(normFilter[norm_thr_rang_ids][-1][0]))
		if sedNormFactor == 0:
		return 2, None
		@@ -230,7 +251,6 @@ class Galaxy(MockObject):

		chip_wcs_local = self.chip_wcs.local(self.real_pos)


		big_galaxy = False
		if self.hlr_disk > 3.0 or self.hlr_bulge > 3.0: # Very big galaxy
		big_galaxy = True
		@@ -244,7 +264,8 @@ class Galaxy(MockObject):

		flat_cube = chip.flat_cube

		xOrderSigPlus = {'A':1.3909419820029296,'B':1.4760376591236062,'C':4.035447379743442,'D':5.5684364343742825,'E':16.260021029735388}
		xOrderSigPlus = {'A': 1.3909419820029296, 'B': 1.4760376591236062,
		'C': 4.035447379743442, 'D': 5.5684364343742825, 'E': 16.260021029735388}
		grating_split_pos_chip = 0 + grating_split_pos

		branges = np.zeros([len(bandpass_list), 2])
		@@ -267,10 +288,12 @@ class Galaxy(MockObject):
		brange = branges[i]

		# psf, pos_shear = psf_model.get_PSF(chip=chip, pos_img=pos_img, bandpass=bandpass, folding_threshold=folding_threshold)
		disk = galsim.Sersic(n=self.disk_sersic_idx, half_light_radius=self.hlr_disk, flux=1.0, gsparams=gsp)
		disk = galsim.Sersic(
		n=self.disk_sersic_idx, half_light_radius=self.hlr_disk, flux=1.0, gsparams=gsp)
		disk_shape = galsim.Shear(g1=self.e1_disk, g2=self.e2_disk)
		disk = disk.shear(disk_shape)
		bulge = galsim.Sersic(n=self.bulge_sersic_idx, half_light_radius=self.hlr_bulge, flux=1.0, gsparams=gsp)
		bulge = galsim.Sersic(
		n=self.bulge_sersic_idx, half_light_radius=self.hlr_bulge, flux=1.0, gsparams=gsp)
		bulge_shape = galsim.Shear(g1=self.e1_bulge, g2=self.e2_bulge)
		bulge = bulge.shear(bulge_shape)

		@@ -286,12 +309,13 @@ class Galaxy(MockObject):
		# kfrac = np.random.random()*(1.0 - self.bfrac)
		# gal = self.bfrac * bulge + (1.0 - self.bfrac - kfrac) * disk + kfrac * knots

		gal = gal.withFlux(tel.pupil_area * exptime)
		if fd_shear:
		g1 += fd_shear.g1
		g2 += fd_shear.g2
		gal_shear = galsim.Shear(g1=g1, g2=g2)
		gal = gal.shear(gal_shear)
		gal = gal.magnify(self.mu)
		gal = gal.withFlux(tel.pupil_area * exptime)
		# gal = galsim.Convolve(psf, gal)

		# if not big_galaxy: # Not apply PSF for very big galaxy
		@@ -299,17 +323,19 @@ class Galaxy(MockObject):
		# # if fd_shear is not None:
		# # gal = gal.shear(fd_shear)

		starImg = gal.drawImage(wcs=chip_wcs_local, offset=offset,method = 'real_space')
		starImg = gal.drawImage(
		wcs=chip_wcs_local, offset=offset, method='real_space')

		origin_star = [y_nominal - (starImg.center.y - starImg.ymin),
		x_nominal - (starImg.center.x - starImg.xmin)]
		starImg.setOrigin(0, 0)
		gal_origin = [origin_star[0], origin_star[1]]
		gal_end = [origin_star[0] + starImg.array.shape[0] - 1, origin_star[1] + starImg.array.shape[1] - 1]
		gal_end = [origin_star[0] + starImg.array.shape[0] -
		1, origin_star[1] + starImg.array.shape[1] - 1]

		if gal_origin[1] < grating_split_pos_chip < gal_end[1]:
		subSlitPos = int(grating_split_pos_chip - gal_origin[1] + 1)
		## part img disperse
		# part img disperse

		subImg_p1 = starImg.array[:, 0:subSlitPos]
		star_p1 = galsim.Image(subImg_p1)
		@@ -332,7 +358,8 @@ class Galaxy(MockObject):
		grating_split_pos=grating_split_pos,
		local_wcs=chip_wcs_local, pos_img=pos_img)

		subImg_p2 = starImg.array[:, subSlitPos+1:starImg.array.shape[1]]
		subImg_p2 = starImg.array[:,
		subSlitPos+1:starImg.array.shape[1]]
		star_p2 = galsim.Image(subImg_p2)
		star_p2.setOrigin(0, 0)
		origin_p2 = [origin_star[0], grating_split_pos_chip]
		@@ -391,11 +418,13 @@ class Galaxy(MockObject):
		def getGSObj(self, psf, g1=0, g2=0, flux=None, filt=None, tel=None, exptime=150.):
		if flux == None:
		flux = self.getElectronFluxFilt(filt, tel, exptime)
		disk = galsim.Sersic(n=self.disk_sersic_idx, half_light_radius=self.hlr_disk, flux=1.0)
		disk = galsim.Sersic(n=self.disk_sersic_idx,
		half_light_radius=self.hlr_disk, flux=1.0)
		disk_shape = galsim.Shear(g1=self.e1_disk, g2=self.e2_disk)
		disk = disk.shear(disk_shape)

		bulge = galsim.Sersic(n=self.bulge_sersic_idx, half_light_radius=self.hlr_bulge, flux=1.0)
		bulge = galsim.Sersic(n=self.bulge_sersic_idx,
		half_light_radius=self.hlr_bulge, flux=1.0)
		bulge_shape = galsim.Shear(g1=self.e1_bulge, g2=self.e2_bulge)
		bulge = bulge.shear(bulge_shape)

		@@ -407,5 +436,6 @@ class Galaxy(MockObject):
		return final

		def getObservedEll(self, g1=0, g2=0):
		e1_obs, e2_obs, e_obs, theta = eObs(self.e1_total, self.e2_total, g1, g2)
		e1_obs, e2_obs, e_obs, theta = eObs(
		self.e1_total, self.e2_total, g1, g2)
		return self.e1_total, self.e2_total, g1, g2, e1_obs, e2_obs