Loading .gitignore 0 → 100644 +7 −0 Original line number Diff line number Diff line *.fits *.cat *.log *.list *.png *.pyc *.so No newline at end of file config/config_injection_20231203.yaml 0 → 100644 +99 −0 Original line number Diff line number Diff line --- ############################################### # # Configuration file for CSST object injection # Last modified: 2022/06/19 # ############################################### # n_objects: 500 rotate_objs: NO use_mpi: YES run_name: "test_20231203" project_cycle: 6 run_counter: 1 pos_sampling: type: "HexGrid" # type: "RectGrid" # grid_spacing: 18.5 # arcsec (~250 pixels) grid_spacing: 15 # arcsec (~500 pixels) # type: "uniform" # object_density: 37 # arcmin^-2 output_img_dir: "/share/home/fangyuedong/injection_pipeline/workspace" input_img_list: "/share/home/fangyuedong/injection_pipeline/input_L1_IMG_20231203.list" ############################################### # PSF setting ############################################### psf_setting: # Which PSF model to use: # "Gauss": simple gaussian profile # "Interp": Interpolated PSF from sampled ray-tracing data psf_model: "Interp" # PSF size [arcseconds] # radius of 80% energy encircled # NOTE: only valid for "Gauss" PSF psf_rcont: 0.15 # path to PSF data # NOTE: only valid for "Interp" PSF psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1" ############################################### # Input path setting # (NOTE) Used NGP Catalog for testing ############################################### # Default path settings for NGP footprint simulation data_dir: "/share/simudata/CSSOSDataProductsSims/data/" input_path: cat_dir: "OnOrbitCalibration/CTargets20211231" star_cat: "CT-NGP_r1.8_G28.hdf5" galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5" SED_templates_path: star_SED: "Catalog_20210126/SpecLib.hdf5" galaxy_SED: "Templates/Galaxy/" ############################################### # Instrumental effects setting # (NOTE) Here only used to construct # ObservationSim.Instrument.Chip object # (TODO) Should readout from header ############################################### ins_effects: # switches bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect # values # dark_exptime: 300 # Exposure time for dark current frames [seconds] # flat_exptime: 150 # Exposure time for flat-fielding frames [seconds] # readout_time: 40 # The read-out time for each channel [seconds] # df_strength: 2.3 # Sillicon sensor diffusion strength # bias_level: 500 # bias level [e-/pixel] # gain: 1.1 # Gain # full_well: 90000 # Full well depth [e-] ############################################### # Random seeds ############################################### random_seeds: seed_Av: 121212 # Seed for generating random intrinsic extinction ############################################### # Measurement setting ############################################### measurement_setting: input_img_list: "/share/home/fangyuedong/injection_pipeline/L1_INJECTED_20231203.list" # input_img_list: "/share/home/fangyuedong/injection_pipeline/input_L1_img_MSC_0000000.list" input_wht_list: "/share/home/fangyuedong/injection_pipeline/L1_WHT_20231203.list" input_flg_list: "/share/home/fangyuedong/injection_pipeline/L1_FLG_20231203.list" # input_psf_list: "/share/home/fangyuedong/injection_pipeline/psf_img_MSC_0000000.list" sex_config: "/share/home/fangyuedong/injection_pipeline/config/default.config" sex_param: "/share/home/fangyuedong/injection_pipeline/config/default.param" n_jobs: 18 output_dir: "/share/home/fangyuedong/injection_pipeline/workspace" No newline at end of file evaluation/calculate_completeness_fraction.py +100 −27 Original line number Diff line number Diff line Loading @@ -8,16 +8,30 @@ from ObservationSim.Instrument import Telescope, Filter, FilterParam VC_A = 2.99792458e+18 # speed of light: A/s # def define_options(): # parser = argparse.ArgumentParser() # parser.add_argument('--TU_catalog', dest='TU_catalog', type=str, required=True, # help='path to the (injected) truth catalog') # parser.add_argument('--source_catalog', dest='source_catalog', type=str, required=True, # help='path to the (extracted) injected catalog') # parser.add_argument('--orig_catalog', dest='orig_catalog', type=str, required=True, # help='path to the (extracted) original catalog') # parser.add_argument('--image', dest='image', type=str, required=True, # help='path to the image, used to get the header info') # parser.add_argument('--output_dir', dest='output_dir', type=str, required=False, # default='./workspace', help='output path') # return parser def define_options(): parser = argparse.ArgumentParser() parser.add_argument('--TU_catalog', dest='TU_catalog', type=str, required=True, help='path to the (injected) truth catalog') parser.add_argument('--source_catalog', dest='source_catalog', type=str, required=True, parser.add_argument('--TU_catalog_list', dest='TU_catalog_list', type=str, required=True, help='path to the list of (injected) truth catalog') parser.add_argument('--source_catalog_list', dest='source_catalog_list', type=str, required=True, help='path to the (extracted) injected catalog') parser.add_argument('--orig_catalog', dest='orig_catalog', type=str, required=True, help='path to the (extracted) original catalog') parser.add_argument('--image', dest='image', type=str, required=True, help='path to the image, used to get the header info') # parser.add_argument('--orig_catalog', dest='orig_catalog', type=str, required=True, # help='path to the list of (extracted) original catalog') # parser.add_argument('--image', dest='image', type=str, required=True, # help='path to the image, used to get the header info') parser.add_argument('--output_dir', dest='output_dir', type=str, required=False, default='./workspace', help='output path') return parser Loading Loading @@ -72,13 +86,24 @@ def convert_catalog(catname): fits_filename = os.path.join(data_dir, base_name + '.fits') text_file.write(fits_filename, overwrite=True) def validation_hist(val, idx, name="val", nbins=10, fig_name='detected_counts.png', output_dir='./'): def validation_hist(val, idx, name="val", nbins=10, bins=None, fig_name='detected_counts.png', output_dir='./', create_figure=True): if bins is None: counts, bins = np.histogram(val, bins=nbins) else: counts, bins = np.histogram(val, bins=bins) is_empty = np.full(len(val), False) for i in range(len(idx)): if idx[i].size == 0: is_empty[i] = True if bins is None: counts_detected, _ = np.histogram(val[~is_empty], bins=nbins) else: counts_detected, _ = np.histogram(val[~is_empty], bins=bins) if create_figure: create_hist_figure(counts, counts_detected, bins, name, output_dir, fig_name) return counts, counts_detected, bins def create_hist_figure(counts, counts_detected, bins, name="val", output_dir='./', fig_name='detected_counts.png'): plt.figure() plt.stairs(counts, bins, color='r', label='TU objects') plt.stairs(counts_detected, bins, color='g', label='Detected') Loading @@ -87,22 +112,38 @@ def validation_hist(val, idx, name="val", nbins=10, fig_name='detected_counts.pn plt.legend(loc='upper right', fancybox=True) fig_name = os.path.join(output_dir, fig_name) plt.savefig(fig_name) return counts, bins def hist_fraction(val, idx, name='val', nbins=10, normed=False, output_dir='./'): def hist_fraction(val, idx, name='val', nbins=10, bins=None, output_dir='./', fig_name="completeness_fraction.png"): if bins is None: counts, bins = np.histogram(val, bins=nbins) else: counts, bins = np.histogram(val, bins=bins) is_empty = np.full(len(val), False) for i in range(len(idx)): if idx[i].size == 0: is_empty[i] = True counts_detected, _ = np.histogram(val[~is_empty], bins=nbins, density=normed) if bins is None: counts_detected, _ = np.histogram(val[~is_empty], bins=nbins) else: counts_detected, _ = np.histogram(val[~is_empty], bins=bins) fraction = counts_detected / counts fraction[np.where(np.isnan(fraction))[0]] = 0. plt.figure() plt.stairs(fraction, bins, color='r', label='completeness fraction') plt.xlabel(name, size='x-large') plt.title("Completeness Fraction") fig_name = os.path.join(output_dir, "completeness_fraction_%s.png"%(name)) fig_name = os.path.join(output_dir, fig_name) plt.savefig(fig_name) return fraction def create_fraction_figure(counts, counts_detected, bins, name='val', output_dir='./', fig_name="completeness_fraction.png"): fraction = counts_detected / counts fraction[np.where(np.isnan(fraction))[0]] = 0. plt.figure() plt.stairs(fraction, bins, color='r', label='completeness fraction') plt.xlabel(name, size='x-large') plt.title("Completeness Fraction") fig_name = os.path.join(output_dir, fig_name) plt.savefig(fig_name) return fraction Loading @@ -116,6 +157,25 @@ def calculate_fraction(TU_catalog, source_catalog, output_dir, nbins=10): fraction = hist_fraction(val=mag_TU, idx=idx1, name="mag_injected", nbins=10, output_dir=output_dir) return counts, bins, fraction def calculate_fraction_multi_cats(TU_catalog_list, source_catalog_list, output_dir, nbins=10): counts = np.zeros(nbins) counts_detected = np.zeros(nbins) bins = np.linspace(18, 26, num=(nbins+1)) for i in range(len(TU_catalog_list)): TU_catalog = TU_catalog_list[i] source_catalog = source_catalog_list[i] convert_catalog(TU_catalog) x_TU_temp, y_TU_temp, col_list = read_catalog(TU_catalog + '.fits', ext_num=1, ra_name="xImage", dec_name="yImage", col_list=["mag"]) mag_TU_temp = col_list[0] x_source_temp, y_source_temp, _ = read_catalog(source_catalog, ext_num=1, ra_name="X_IMAGE", dec_name="Y_IMAGE") idx1, _, = match_catalogs_img(x1=x_TU_temp, y1=y_TU_temp, x2=x_source_temp, y2=y_source_temp) counts_temp, counts_detected_temp, _ = validation_hist(val=mag_TU_temp, idx=idx1, name="mag_injected", bins=bins, output_dir=output_dir, create_figure=False) counts += counts_temp counts_detected += counts_detected_temp create_hist_figure(counts, counts_detected, bins, "mag_injected", output_dir) fraction = create_fraction_figure(counts, counts_detected, bins, 'mag_injected', output_dir) return counts, counts_detected, bins, fraction def calculate_undetected_flux(orig_cat, mag_bins, fraction, mag_low=20.0, mag_high=26.0, image=None, output_dir='./'): # Get info from original image hdu = fits.open(image) Loading Loading @@ -166,19 +226,32 @@ def calculate_undetected_flux(orig_cat, mag_bins, fraction, mag_low=20.0, mag_hi undetected_flux /= (float(nx_pix) * float(ny_pix)) return undetected_flux # if __name__ == "__main__": # args = define_options().parse_args() # counts, bins, fraction = calculate_fraction( # TU_catalog=args.TU_catalog, # source_catalog=args.source_catalog, # output_dir=args.output_dir, # nbins=20 # ) # undetected_flux = calculate_undetected_flux( # orig_cat=args.orig_catalog, # mag_bins=bins, # fraction=fraction, # image=args.image, # output_dir=args.output_dir, # ) # print(undetected_flux) if __name__ == "__main__": args = define_options().parse_args() counts, bins, fraction = calculate_fraction( TU_catalog=args.TU_catalog, source_catalog=args.source_catalog, with open(args.TU_catalog_list) as file: TU_catalog_list = [line.rstrip() for line in file] with open(args.source_catalog_list) as file: source_catalog_list = [line.rstrip() for line in file] counts, counts_detected, bins, fraction = calculate_fraction_multi_cats( TU_catalog_list=TU_catalog_list, source_catalog_list=source_catalog_list, output_dir=args.output_dir, nbins=20 ) No newline at end of file undetected_flux = calculate_undetected_flux( orig_cat=args.orig_catalog, mag_bins=bins, fraction=fraction, image=args.image, output_dir=args.output_dir, ) print(undetected_flux) No newline at end of file evaluation/cross_match_catalogs.py +5 −19 Original line number Diff line number Diff line import numpy as np import astropy.units as u import matplotlib.pyplot as plt from astropy.coordinates import SkyCoord from astropy.io import fits from astropy.io import ascii from sklearn.neighbors import BallTree # TU_catalog = "test_RectGrid_20220628.cat" # source_catalog = "extracted_test_RectGrid_20220628.fits" TU_catalog = "injected_bkgsub_img.cat" source_catalog = "extracted_injected_bkgsub_img.fits" Loading @@ -26,7 +23,6 @@ def read_catalog(catname, ext_num=1, ra_name='ra', dec_name='dec', col_list=[]): if len(col_list) > 0: for col in col_list: col_other.append(data[col]) # print(ra, dec) return ra, dec, col_other def match_catalogs_sky(ra1, dec1, ra2, dec2, max_dist=0.6, others1=[], others2=[], thresh=[]): Loading @@ -37,33 +33,24 @@ def match_catalogs_sky(ra1, dec1, ra2, dec2, max_dist=0.6, others1=[], others2=[ # print(idx2) # print(np.shape(idx1)) # print(np.shape(idx2)) # TODO def match_catalogs_img(x1, y1, x2, y2, max_dist=0.5, others1=[], others2=[], thresh=[]): def match_catalogs_img(x1, y1, x2, y2, max_dist=2, others1=[], others2=[], thresh=[]): cat1 = np.array([(x, y) for x,y in zip(x1, y1)]) cat2 = np.array([(x, y) for x,y in zip(x2, y2)]) # print(np.shape(cat1)) # print(np.shape(cat2)) tree = BallTree(cat2) idx1 = tree.query_radius(cat1, r = max_dist) tree = BallTree(cat1) idx2 = tree.query_radius(cat2, r = max_dist) # print(np.shape(idx1)) tot = 0 print(idx1) for idx in idx1: if len(idx) == 0: continue if len(idx) > 1: print(len(idx)) tot += 1 print(tot) print("number of matched sources = ", tot) return idx1, idx2 def validation_hist(val1, idx1, name="val1", nbins=10): counts, bins = np.histogram(val1) plt.stairs(counts, bins) plt.xlabel(name, size='x-large') plt.savefig("detection_completeness.png") # plt.show() if __name__=="__main__": convert_catalog(TU_catalog) ra_TU, dec_TU, _ = read_catalog('test_ascii_to_fits.fits', ext_num=1, ra_name="ra", dec_name="dec") Loading @@ -74,4 +61,3 @@ if __name__=="__main__": # match_catalogs_sky(ra1=ra_TU, dec1=dec_TU, ra2=ra_source, dec2=dec_source) idx1, idx2, = match_catalogs_img(x1=x_TU, y1=y_TU, x2=x_source, y2=y_source) # print(ra_TU, dec_TU) No newline at end of file validation_hist(mag_TU, idx1, name="mag_injected") No newline at end of file injection_pipeline/Catalog/C6_SimCat.py 0 → 100644 +232 −0 Original line number Diff line number Diff line import os import galsim import random import numpy as np import h5py as h5 import healpy as hp import astropy.constants as cons import traceback from astropy.coordinates import spherical_to_cartesian from astropy.table import Table from scipy import interpolate from datetime import datetime from ObservationSim.MockObject import CatalogBase, Star, Galaxy, Quasar from ObservationSim.MockObject._util import tag_sed, getObservedSED, getABMAG, integrate_sed_bandpass, comoving_dist from ObservationSim.Astrometry.Astrometry_util import on_orbit_obs_position # (TEST) from astropy.cosmology import FlatLambdaCDM from astropy import constants from astropy import units as U try: import importlib.resources as pkg_resources except ImportError: # Try backported to PY<37 'importlib_resources' import importlib_resources as pkg_resources # CONSTANTS NSIDE = 128 def get_bundleIndex(healpixID_ring, bundleOrder=4, healpixOrder=7): assert NSIDE == 2**healpixOrder shift = healpixOrder - bundleOrder shift = 2*shift nside_bundle = 2**bundleOrder nside_healpix= 2**healpixOrder healpixID_nest= hp.ring2nest(nside_healpix, healpixID_ring) bundleID_nest = (healpixID_nest >> shift) bundleID_ring = hp.nest2ring(nside_bundle, bundleID_nest) return bundleID_ring class SimCat(CatalogBase): def __init__(self, config, chip, nobjects=None): super().__init__() self.cat_dir = os.path.join(config["data_dir"], config["catalog_options"]["input_path"]["cat_dir"]) self.seed_Av = config["catalog_options"]["seed_Av"] self.cosmo = FlatLambdaCDM(H0=67.66, Om0=0.3111) with pkg_resources.path('Catalog.data', 'SLOAN_SDSS.g.fits') as filter_path: self.normF_star = Table.read(str(filter_path)) self.config = config self.chip = chip galaxy_dir = config["catalog_options"]["input_path"]["galaxy_cat"] self.galaxy_path = os.path.join(self.cat_dir, galaxy_dir) self.galaxy_SED_path = os.path.join(config["data_dir"], config["catalog_options"]["SED_templates_path"]["galaxy_SED"]) self._load_SED_lib_gals() if "rotateEll" in config["catalog_options"]: self.rotation = float(int(config["catalog_options"]["rotateEll"]/45.)) else: self.rotation = 0. self._get_healpix_list() self._load(nobjects=nobjects) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) ra_min, ra_max, dec_min, dec_max = self.sky_coverage.xmin, self.sky_coverage.xmax, self.sky_coverage.ymin, self.sky_coverage.ymax ra = np.deg2rad(np.array([ra_min, ra_max, ra_max, ra_min])) dec = np.deg2rad(np.array([dec_max, dec_max, dec_min, dec_min])) self.pix_list = hp.query_polygon( NSIDE, hp.ang2vec(np.radians(90.) - dec, ra), inclusive=True ) if self.logger is not None: msg = str(("HEALPix List: ", self.pix_list)) self.logger.info(msg) else: print("HEALPix List: ", self.pix_list) def load_norm_filt(self, obj): if obj.type == "star": return self.normF_star elif obj.type == "galaxy" or obj.type == "quasar": return None else: return None def _load_SED_lib_gals(self): pcs = h5.File(os.path.join(self.galaxy_SED_path, "pcs.h5"), "r") lamb = h5.File(os.path.join(self.galaxy_SED_path, "lamb.h5"), "r") self.lamb_gal = lamb['lamb'][()] self.pcs = pcs['pcs'][()] def _load_gals(self, gals, pix_id=None, cat_id=0, nobjects=1): # Load how mnay objects? max_ngals = len(gals['ra']) remain = nobjects for igals in range(max_ngals): if remain == 0: break param = self.initialize_param() param['ra'] = ra_arr[igals] param['dec'] = dec_arr[igals] param['ra_orig'] = gals['ra'][igals] param['dec_orig'] = gals['dec'][igals] # [TODO] param['mag_use_normal'] = gals['mag_csst_%s'%(self.filt.filter_type)][igals] # if self.filt.is_too_dim(mag=param['mag_use_normal'], margin=self.config["obs_setting"]["mag_lim_margin"]): # continue param['z'] = gals['redshift'][igals] param['model_tag'] = 'None' param['g1'] = gals['shear'][igals][0] param['g2'] = gals['shear'][igals][1] param['kappa'] = gals['kappa'][igals] param['e1'] = gals['ellipticity_true'][igals][0] param['e2'] = gals['ellipticity_true'][igals][1] # For shape calculation param['ell_total'] = np.sqrt(param['e1']**2 + param['e2']**2) if param['ell_total'] > 0.9: continue remain -= 1 param['e1_disk'] = param['e1'] param['e2_disk'] = param['e2'] param['e1_bulge'] = param['e1'] param['e2_bulge'] = param['e2'] param['delta_ra'] = 0 param['delta_dec'] = 0 # Masses param['bulgemass'] = gals['bulgemass'][igals] param['diskmass'] = gals['diskmass'][igals] param['size'] = gals['size'][igals] if param['size'] > self.max_size: self.max_size = param['size'] # Sersic index param['disk_sersic_idx'] = 1. param['bulge_sersic_idx'] = 4. # Sizes param['bfrac'] = param['bulgemass']/(param['bulgemass'] + param['diskmass']) if param['bfrac'] >= 0.6: param['hlr_bulge'] = param['size'] param['hlr_disk'] = param['size'] * (1. - param['bfrac']) else: param['hlr_disk'] = param['size'] param['hlr_bulge'] = param['size'] * param['bfrac'] # SED coefficients param['coeff'] = gals['coeff'][igals] param['detA'] = gals['detA'][igals] # Others param['galType'] = gals['type'][igals] param['veldisp'] = gals['veldisp'][igals] # TEST no redening and no extinction param['av'] = 0.0 param['redden'] = 0 param['star'] = 0 # Galaxy # TEMP self.ids += 1 # param['id'] = self.ids param['id'] = '%06d'%(int(pix_id)) + '%06d'%(cat_id) + '%08d'%(igals) if param['star'] == 0: obj = Galaxy(param, self.rotation, logger=self.logger) self.objs.append(obj) return remain def _load(self, nobjects=1): from itertools import cycle self.objs = [] self.ids = 0 to_be_read_in = nobjects pool = cycle(self.pix_list) for pix in pool: try: if to_be_read_in == 0: break bundleID = get_bundleIndex(pix) file_path = os.path.join(self.galaxy_path, "galaxies_C6_bundle{:06}.h5".format(bundleID)) gals_cat = h5.File(file_path, 'r')['galaxies'] gals = gals_cat[str(pix)] to_be_read_in = self._load_gals(gals, pix_id=pix, cat_id=bundleID, n_objects=to_be_read_in) del gals except Exception as e: traceback.print_exc() print(e) def load_sed(self, obj, **kwargs): factor = 10**(-.4 * self.cosmo.distmod(obj.z).value) flux = np.matmul(self.pcs, obj.coeff) * factor # if np.any(flux < 0): # raise ValueError("Glaxy %s: negative SED fluxes"%obj.id) flux[flux < 0] = 0. sedcat = np.vstack((self.lamb_gal, flux)).T sed_data = getObservedSED( sedCat=sedcat, redshift=obj.z, av=obj.param["av"], redden=obj.param["redden"] ) wave, flux = sed_data[0], sed_data[1] speci = interpolate.interp1d(wave, flux) lamb = np.arange(2000, 11001+0.5, 0.5) y = speci(lamb) # erg/s/cm2/A --> photon/s/m2/A all_sed = y * lamb / (cons.h.value * cons.c.value) * 1e-13 sed = Table(np.array([lamb, all_sed]).T, names=('WAVELENGTH', 'FLUX')) del wave del flux return sed No newline at end of file Loading
.gitignore 0 → 100644 +7 −0 Original line number Diff line number Diff line *.fits *.cat *.log *.list *.png *.pyc *.so No newline at end of file
config/config_injection_20231203.yaml 0 → 100644 +99 −0 Original line number Diff line number Diff line --- ############################################### # # Configuration file for CSST object injection # Last modified: 2022/06/19 # ############################################### # n_objects: 500 rotate_objs: NO use_mpi: YES run_name: "test_20231203" project_cycle: 6 run_counter: 1 pos_sampling: type: "HexGrid" # type: "RectGrid" # grid_spacing: 18.5 # arcsec (~250 pixels) grid_spacing: 15 # arcsec (~500 pixels) # type: "uniform" # object_density: 37 # arcmin^-2 output_img_dir: "/share/home/fangyuedong/injection_pipeline/workspace" input_img_list: "/share/home/fangyuedong/injection_pipeline/input_L1_IMG_20231203.list" ############################################### # PSF setting ############################################### psf_setting: # Which PSF model to use: # "Gauss": simple gaussian profile # "Interp": Interpolated PSF from sampled ray-tracing data psf_model: "Interp" # PSF size [arcseconds] # radius of 80% energy encircled # NOTE: only valid for "Gauss" PSF psf_rcont: 0.15 # path to PSF data # NOTE: only valid for "Interp" PSF psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1" ############################################### # Input path setting # (NOTE) Used NGP Catalog for testing ############################################### # Default path settings for NGP footprint simulation data_dir: "/share/simudata/CSSOSDataProductsSims/data/" input_path: cat_dir: "OnOrbitCalibration/CTargets20211231" star_cat: "CT-NGP_r1.8_G28.hdf5" galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5" SED_templates_path: star_SED: "Catalog_20210126/SpecLib.hdf5" galaxy_SED: "Templates/Galaxy/" ############################################### # Instrumental effects setting # (NOTE) Here only used to construct # ObservationSim.Instrument.Chip object # (TODO) Should readout from header ############################################### ins_effects: # switches bright_fatter: ON # Whether to simulate Brighter-Fatter (also diffusion) effect # values # dark_exptime: 300 # Exposure time for dark current frames [seconds] # flat_exptime: 150 # Exposure time for flat-fielding frames [seconds] # readout_time: 40 # The read-out time for each channel [seconds] # df_strength: 2.3 # Sillicon sensor diffusion strength # bias_level: 500 # bias level [e-/pixel] # gain: 1.1 # Gain # full_well: 90000 # Full well depth [e-] ############################################### # Random seeds ############################################### random_seeds: seed_Av: 121212 # Seed for generating random intrinsic extinction ############################################### # Measurement setting ############################################### measurement_setting: input_img_list: "/share/home/fangyuedong/injection_pipeline/L1_INJECTED_20231203.list" # input_img_list: "/share/home/fangyuedong/injection_pipeline/input_L1_img_MSC_0000000.list" input_wht_list: "/share/home/fangyuedong/injection_pipeline/L1_WHT_20231203.list" input_flg_list: "/share/home/fangyuedong/injection_pipeline/L1_FLG_20231203.list" # input_psf_list: "/share/home/fangyuedong/injection_pipeline/psf_img_MSC_0000000.list" sex_config: "/share/home/fangyuedong/injection_pipeline/config/default.config" sex_param: "/share/home/fangyuedong/injection_pipeline/config/default.param" n_jobs: 18 output_dir: "/share/home/fangyuedong/injection_pipeline/workspace" No newline at end of file
evaluation/calculate_completeness_fraction.py +100 −27 Original line number Diff line number Diff line Loading @@ -8,16 +8,30 @@ from ObservationSim.Instrument import Telescope, Filter, FilterParam VC_A = 2.99792458e+18 # speed of light: A/s # def define_options(): # parser = argparse.ArgumentParser() # parser.add_argument('--TU_catalog', dest='TU_catalog', type=str, required=True, # help='path to the (injected) truth catalog') # parser.add_argument('--source_catalog', dest='source_catalog', type=str, required=True, # help='path to the (extracted) injected catalog') # parser.add_argument('--orig_catalog', dest='orig_catalog', type=str, required=True, # help='path to the (extracted) original catalog') # parser.add_argument('--image', dest='image', type=str, required=True, # help='path to the image, used to get the header info') # parser.add_argument('--output_dir', dest='output_dir', type=str, required=False, # default='./workspace', help='output path') # return parser def define_options(): parser = argparse.ArgumentParser() parser.add_argument('--TU_catalog', dest='TU_catalog', type=str, required=True, help='path to the (injected) truth catalog') parser.add_argument('--source_catalog', dest='source_catalog', type=str, required=True, parser.add_argument('--TU_catalog_list', dest='TU_catalog_list', type=str, required=True, help='path to the list of (injected) truth catalog') parser.add_argument('--source_catalog_list', dest='source_catalog_list', type=str, required=True, help='path to the (extracted) injected catalog') parser.add_argument('--orig_catalog', dest='orig_catalog', type=str, required=True, help='path to the (extracted) original catalog') parser.add_argument('--image', dest='image', type=str, required=True, help='path to the image, used to get the header info') # parser.add_argument('--orig_catalog', dest='orig_catalog', type=str, required=True, # help='path to the list of (extracted) original catalog') # parser.add_argument('--image', dest='image', type=str, required=True, # help='path to the image, used to get the header info') parser.add_argument('--output_dir', dest='output_dir', type=str, required=False, default='./workspace', help='output path') return parser Loading Loading @@ -72,13 +86,24 @@ def convert_catalog(catname): fits_filename = os.path.join(data_dir, base_name + '.fits') text_file.write(fits_filename, overwrite=True) def validation_hist(val, idx, name="val", nbins=10, fig_name='detected_counts.png', output_dir='./'): def validation_hist(val, idx, name="val", nbins=10, bins=None, fig_name='detected_counts.png', output_dir='./', create_figure=True): if bins is None: counts, bins = np.histogram(val, bins=nbins) else: counts, bins = np.histogram(val, bins=bins) is_empty = np.full(len(val), False) for i in range(len(idx)): if idx[i].size == 0: is_empty[i] = True if bins is None: counts_detected, _ = np.histogram(val[~is_empty], bins=nbins) else: counts_detected, _ = np.histogram(val[~is_empty], bins=bins) if create_figure: create_hist_figure(counts, counts_detected, bins, name, output_dir, fig_name) return counts, counts_detected, bins def create_hist_figure(counts, counts_detected, bins, name="val", output_dir='./', fig_name='detected_counts.png'): plt.figure() plt.stairs(counts, bins, color='r', label='TU objects') plt.stairs(counts_detected, bins, color='g', label='Detected') Loading @@ -87,22 +112,38 @@ def validation_hist(val, idx, name="val", nbins=10, fig_name='detected_counts.pn plt.legend(loc='upper right', fancybox=True) fig_name = os.path.join(output_dir, fig_name) plt.savefig(fig_name) return counts, bins def hist_fraction(val, idx, name='val', nbins=10, normed=False, output_dir='./'): def hist_fraction(val, idx, name='val', nbins=10, bins=None, output_dir='./', fig_name="completeness_fraction.png"): if bins is None: counts, bins = np.histogram(val, bins=nbins) else: counts, bins = np.histogram(val, bins=bins) is_empty = np.full(len(val), False) for i in range(len(idx)): if idx[i].size == 0: is_empty[i] = True counts_detected, _ = np.histogram(val[~is_empty], bins=nbins, density=normed) if bins is None: counts_detected, _ = np.histogram(val[~is_empty], bins=nbins) else: counts_detected, _ = np.histogram(val[~is_empty], bins=bins) fraction = counts_detected / counts fraction[np.where(np.isnan(fraction))[0]] = 0. plt.figure() plt.stairs(fraction, bins, color='r', label='completeness fraction') plt.xlabel(name, size='x-large') plt.title("Completeness Fraction") fig_name = os.path.join(output_dir, "completeness_fraction_%s.png"%(name)) fig_name = os.path.join(output_dir, fig_name) plt.savefig(fig_name) return fraction def create_fraction_figure(counts, counts_detected, bins, name='val', output_dir='./', fig_name="completeness_fraction.png"): fraction = counts_detected / counts fraction[np.where(np.isnan(fraction))[0]] = 0. plt.figure() plt.stairs(fraction, bins, color='r', label='completeness fraction') plt.xlabel(name, size='x-large') plt.title("Completeness Fraction") fig_name = os.path.join(output_dir, fig_name) plt.savefig(fig_name) return fraction Loading @@ -116,6 +157,25 @@ def calculate_fraction(TU_catalog, source_catalog, output_dir, nbins=10): fraction = hist_fraction(val=mag_TU, idx=idx1, name="mag_injected", nbins=10, output_dir=output_dir) return counts, bins, fraction def calculate_fraction_multi_cats(TU_catalog_list, source_catalog_list, output_dir, nbins=10): counts = np.zeros(nbins) counts_detected = np.zeros(nbins) bins = np.linspace(18, 26, num=(nbins+1)) for i in range(len(TU_catalog_list)): TU_catalog = TU_catalog_list[i] source_catalog = source_catalog_list[i] convert_catalog(TU_catalog) x_TU_temp, y_TU_temp, col_list = read_catalog(TU_catalog + '.fits', ext_num=1, ra_name="xImage", dec_name="yImage", col_list=["mag"]) mag_TU_temp = col_list[0] x_source_temp, y_source_temp, _ = read_catalog(source_catalog, ext_num=1, ra_name="X_IMAGE", dec_name="Y_IMAGE") idx1, _, = match_catalogs_img(x1=x_TU_temp, y1=y_TU_temp, x2=x_source_temp, y2=y_source_temp) counts_temp, counts_detected_temp, _ = validation_hist(val=mag_TU_temp, idx=idx1, name="mag_injected", bins=bins, output_dir=output_dir, create_figure=False) counts += counts_temp counts_detected += counts_detected_temp create_hist_figure(counts, counts_detected, bins, "mag_injected", output_dir) fraction = create_fraction_figure(counts, counts_detected, bins, 'mag_injected', output_dir) return counts, counts_detected, bins, fraction def calculate_undetected_flux(orig_cat, mag_bins, fraction, mag_low=20.0, mag_high=26.0, image=None, output_dir='./'): # Get info from original image hdu = fits.open(image) Loading Loading @@ -166,19 +226,32 @@ def calculate_undetected_flux(orig_cat, mag_bins, fraction, mag_low=20.0, mag_hi undetected_flux /= (float(nx_pix) * float(ny_pix)) return undetected_flux # if __name__ == "__main__": # args = define_options().parse_args() # counts, bins, fraction = calculate_fraction( # TU_catalog=args.TU_catalog, # source_catalog=args.source_catalog, # output_dir=args.output_dir, # nbins=20 # ) # undetected_flux = calculate_undetected_flux( # orig_cat=args.orig_catalog, # mag_bins=bins, # fraction=fraction, # image=args.image, # output_dir=args.output_dir, # ) # print(undetected_flux) if __name__ == "__main__": args = define_options().parse_args() counts, bins, fraction = calculate_fraction( TU_catalog=args.TU_catalog, source_catalog=args.source_catalog, with open(args.TU_catalog_list) as file: TU_catalog_list = [line.rstrip() for line in file] with open(args.source_catalog_list) as file: source_catalog_list = [line.rstrip() for line in file] counts, counts_detected, bins, fraction = calculate_fraction_multi_cats( TU_catalog_list=TU_catalog_list, source_catalog_list=source_catalog_list, output_dir=args.output_dir, nbins=20 ) No newline at end of file undetected_flux = calculate_undetected_flux( orig_cat=args.orig_catalog, mag_bins=bins, fraction=fraction, image=args.image, output_dir=args.output_dir, ) print(undetected_flux) No newline at end of file
evaluation/cross_match_catalogs.py +5 −19 Original line number Diff line number Diff line import numpy as np import astropy.units as u import matplotlib.pyplot as plt from astropy.coordinates import SkyCoord from astropy.io import fits from astropy.io import ascii from sklearn.neighbors import BallTree # TU_catalog = "test_RectGrid_20220628.cat" # source_catalog = "extracted_test_RectGrid_20220628.fits" TU_catalog = "injected_bkgsub_img.cat" source_catalog = "extracted_injected_bkgsub_img.fits" Loading @@ -26,7 +23,6 @@ def read_catalog(catname, ext_num=1, ra_name='ra', dec_name='dec', col_list=[]): if len(col_list) > 0: for col in col_list: col_other.append(data[col]) # print(ra, dec) return ra, dec, col_other def match_catalogs_sky(ra1, dec1, ra2, dec2, max_dist=0.6, others1=[], others2=[], thresh=[]): Loading @@ -37,33 +33,24 @@ def match_catalogs_sky(ra1, dec1, ra2, dec2, max_dist=0.6, others1=[], others2=[ # print(idx2) # print(np.shape(idx1)) # print(np.shape(idx2)) # TODO def match_catalogs_img(x1, y1, x2, y2, max_dist=0.5, others1=[], others2=[], thresh=[]): def match_catalogs_img(x1, y1, x2, y2, max_dist=2, others1=[], others2=[], thresh=[]): cat1 = np.array([(x, y) for x,y in zip(x1, y1)]) cat2 = np.array([(x, y) for x,y in zip(x2, y2)]) # print(np.shape(cat1)) # print(np.shape(cat2)) tree = BallTree(cat2) idx1 = tree.query_radius(cat1, r = max_dist) tree = BallTree(cat1) idx2 = tree.query_radius(cat2, r = max_dist) # print(np.shape(idx1)) tot = 0 print(idx1) for idx in idx1: if len(idx) == 0: continue if len(idx) > 1: print(len(idx)) tot += 1 print(tot) print("number of matched sources = ", tot) return idx1, idx2 def validation_hist(val1, idx1, name="val1", nbins=10): counts, bins = np.histogram(val1) plt.stairs(counts, bins) plt.xlabel(name, size='x-large') plt.savefig("detection_completeness.png") # plt.show() if __name__=="__main__": convert_catalog(TU_catalog) ra_TU, dec_TU, _ = read_catalog('test_ascii_to_fits.fits', ext_num=1, ra_name="ra", dec_name="dec") Loading @@ -74,4 +61,3 @@ if __name__=="__main__": # match_catalogs_sky(ra1=ra_TU, dec1=dec_TU, ra2=ra_source, dec2=dec_source) idx1, idx2, = match_catalogs_img(x1=x_TU, y1=y_TU, x2=x_source, y2=y_source) # print(ra_TU, dec_TU) No newline at end of file validation_hist(mag_TU, idx1, name="mag_injected") No newline at end of file
injection_pipeline/Catalog/C6_SimCat.py 0 → 100644 +232 −0 Original line number Diff line number Diff line import os import galsim import random import numpy as np import h5py as h5 import healpy as hp import astropy.constants as cons import traceback from astropy.coordinates import spherical_to_cartesian from astropy.table import Table from scipy import interpolate from datetime import datetime from ObservationSim.MockObject import CatalogBase, Star, Galaxy, Quasar from ObservationSim.MockObject._util import tag_sed, getObservedSED, getABMAG, integrate_sed_bandpass, comoving_dist from ObservationSim.Astrometry.Astrometry_util import on_orbit_obs_position # (TEST) from astropy.cosmology import FlatLambdaCDM from astropy import constants from astropy import units as U try: import importlib.resources as pkg_resources except ImportError: # Try backported to PY<37 'importlib_resources' import importlib_resources as pkg_resources # CONSTANTS NSIDE = 128 def get_bundleIndex(healpixID_ring, bundleOrder=4, healpixOrder=7): assert NSIDE == 2**healpixOrder shift = healpixOrder - bundleOrder shift = 2*shift nside_bundle = 2**bundleOrder nside_healpix= 2**healpixOrder healpixID_nest= hp.ring2nest(nside_healpix, healpixID_ring) bundleID_nest = (healpixID_nest >> shift) bundleID_ring = hp.nest2ring(nside_bundle, bundleID_nest) return bundleID_ring class SimCat(CatalogBase): def __init__(self, config, chip, nobjects=None): super().__init__() self.cat_dir = os.path.join(config["data_dir"], config["catalog_options"]["input_path"]["cat_dir"]) self.seed_Av = config["catalog_options"]["seed_Av"] self.cosmo = FlatLambdaCDM(H0=67.66, Om0=0.3111) with pkg_resources.path('Catalog.data', 'SLOAN_SDSS.g.fits') as filter_path: self.normF_star = Table.read(str(filter_path)) self.config = config self.chip = chip galaxy_dir = config["catalog_options"]["input_path"]["galaxy_cat"] self.galaxy_path = os.path.join(self.cat_dir, galaxy_dir) self.galaxy_SED_path = os.path.join(config["data_dir"], config["catalog_options"]["SED_templates_path"]["galaxy_SED"]) self._load_SED_lib_gals() if "rotateEll" in config["catalog_options"]: self.rotation = float(int(config["catalog_options"]["rotateEll"]/45.)) else: self.rotation = 0. self._get_healpix_list() self._load(nobjects=nobjects) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) ra_min, ra_max, dec_min, dec_max = self.sky_coverage.xmin, self.sky_coverage.xmax, self.sky_coverage.ymin, self.sky_coverage.ymax ra = np.deg2rad(np.array([ra_min, ra_max, ra_max, ra_min])) dec = np.deg2rad(np.array([dec_max, dec_max, dec_min, dec_min])) self.pix_list = hp.query_polygon( NSIDE, hp.ang2vec(np.radians(90.) - dec, ra), inclusive=True ) if self.logger is not None: msg = str(("HEALPix List: ", self.pix_list)) self.logger.info(msg) else: print("HEALPix List: ", self.pix_list) def load_norm_filt(self, obj): if obj.type == "star": return self.normF_star elif obj.type == "galaxy" or obj.type == "quasar": return None else: return None def _load_SED_lib_gals(self): pcs = h5.File(os.path.join(self.galaxy_SED_path, "pcs.h5"), "r") lamb = h5.File(os.path.join(self.galaxy_SED_path, "lamb.h5"), "r") self.lamb_gal = lamb['lamb'][()] self.pcs = pcs['pcs'][()] def _load_gals(self, gals, pix_id=None, cat_id=0, nobjects=1): # Load how mnay objects? max_ngals = len(gals['ra']) remain = nobjects for igals in range(max_ngals): if remain == 0: break param = self.initialize_param() param['ra'] = ra_arr[igals] param['dec'] = dec_arr[igals] param['ra_orig'] = gals['ra'][igals] param['dec_orig'] = gals['dec'][igals] # [TODO] param['mag_use_normal'] = gals['mag_csst_%s'%(self.filt.filter_type)][igals] # if self.filt.is_too_dim(mag=param['mag_use_normal'], margin=self.config["obs_setting"]["mag_lim_margin"]): # continue param['z'] = gals['redshift'][igals] param['model_tag'] = 'None' param['g1'] = gals['shear'][igals][0] param['g2'] = gals['shear'][igals][1] param['kappa'] = gals['kappa'][igals] param['e1'] = gals['ellipticity_true'][igals][0] param['e2'] = gals['ellipticity_true'][igals][1] # For shape calculation param['ell_total'] = np.sqrt(param['e1']**2 + param['e2']**2) if param['ell_total'] > 0.9: continue remain -= 1 param['e1_disk'] = param['e1'] param['e2_disk'] = param['e2'] param['e1_bulge'] = param['e1'] param['e2_bulge'] = param['e2'] param['delta_ra'] = 0 param['delta_dec'] = 0 # Masses param['bulgemass'] = gals['bulgemass'][igals] param['diskmass'] = gals['diskmass'][igals] param['size'] = gals['size'][igals] if param['size'] > self.max_size: self.max_size = param['size'] # Sersic index param['disk_sersic_idx'] = 1. param['bulge_sersic_idx'] = 4. # Sizes param['bfrac'] = param['bulgemass']/(param['bulgemass'] + param['diskmass']) if param['bfrac'] >= 0.6: param['hlr_bulge'] = param['size'] param['hlr_disk'] = param['size'] * (1. - param['bfrac']) else: param['hlr_disk'] = param['size'] param['hlr_bulge'] = param['size'] * param['bfrac'] # SED coefficients param['coeff'] = gals['coeff'][igals] param['detA'] = gals['detA'][igals] # Others param['galType'] = gals['type'][igals] param['veldisp'] = gals['veldisp'][igals] # TEST no redening and no extinction param['av'] = 0.0 param['redden'] = 0 param['star'] = 0 # Galaxy # TEMP self.ids += 1 # param['id'] = self.ids param['id'] = '%06d'%(int(pix_id)) + '%06d'%(cat_id) + '%08d'%(igals) if param['star'] == 0: obj = Galaxy(param, self.rotation, logger=self.logger) self.objs.append(obj) return remain def _load(self, nobjects=1): from itertools import cycle self.objs = [] self.ids = 0 to_be_read_in = nobjects pool = cycle(self.pix_list) for pix in pool: try: if to_be_read_in == 0: break bundleID = get_bundleIndex(pix) file_path = os.path.join(self.galaxy_path, "galaxies_C6_bundle{:06}.h5".format(bundleID)) gals_cat = h5.File(file_path, 'r')['galaxies'] gals = gals_cat[str(pix)] to_be_read_in = self._load_gals(gals, pix_id=pix, cat_id=bundleID, n_objects=to_be_read_in) del gals except Exception as e: traceback.print_exc() print(e) def load_sed(self, obj, **kwargs): factor = 10**(-.4 * self.cosmo.distmod(obj.z).value) flux = np.matmul(self.pcs, obj.coeff) * factor # if np.any(flux < 0): # raise ValueError("Glaxy %s: negative SED fluxes"%obj.id) flux[flux < 0] = 0. sedcat = np.vstack((self.lamb_gal, flux)).T sed_data = getObservedSED( sedCat=sedcat, redshift=obj.z, av=obj.param["av"], redden=obj.param["redden"] ) wave, flux = sed_data[0], sed_data[1] speci = interpolate.interp1d(wave, flux) lamb = np.arange(2000, 11001+0.5, 0.5) y = speci(lamb) # erg/s/cm2/A --> photon/s/m2/A all_sed = y * lamb / (cons.h.value * cons.c.value) * 1e-13 sed = Table(np.array([lamb, all_sed]).T, names=('WAVELENGTH', 'FLUX')) del wave del flux return sed No newline at end of file