Commit de19281e authored by Fang Yuedong's avatar Fang Yuedong
Browse files

backup

parent fbaecf55
Loading
Loading
Loading
Loading

.gitignore

0 → 100644
+7 −0
Original line number Diff line number Diff line
*.fits
*.cat
*.log
*.list
*.png
*.pyc
*.so
 No newline at end of file
+99 −0
Original line number Diff line number Diff line
---
###############################################
#
#  Configuration file for CSST object injection
#      Last modified: 2022/06/19
#
###############################################
# n_objects: 500
rotate_objs: NO
use_mpi: YES
run_name: "test_20231203"
project_cycle: 6
run_counter: 1

pos_sampling:
  type: "HexGrid"
  # type: "RectGrid"
  # grid_spacing: 18.5 # arcsec (~250 pixels)
  grid_spacing: 15 # arcsec (~500 pixels)
  # type: "uniform"
  # object_density: 37 # arcmin^-2

output_img_dir: "/share/home/fangyuedong/injection_pipeline/workspace"
input_img_list: "/share/home/fangyuedong/injection_pipeline/input_L1_IMG_20231203.list"

###############################################
# PSF setting
###############################################
psf_setting:
  
  # Which PSF model to use:
  # "Gauss": simple gaussian profile
  # "Interp": Interpolated PSF from sampled ray-tracing data
  psf_model: "Interp"

  # PSF size [arcseconds]
  # radius of 80% energy encircled
  # NOTE: only valid for "Gauss" PSF
  psf_rcont: 0.15

  # path to PSF data
  # NOTE: only valid for "Interp" PSF
  psf_dir: "/share/simudata/CSSOSDataProductsSims/data/psfCube1"

###############################################
# Input path setting
# (NOTE) Used NGP Catalog for testing
###############################################
# Default path settings for NGP footprint simulation
data_dir: "/share/simudata/CSSOSDataProductsSims/data/"

input_path:
  cat_dir: "OnOrbitCalibration/CTargets20211231"
  star_cat: "CT-NGP_r1.8_G28.hdf5"
  galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5"

SED_templates_path:
  star_SED: "Catalog_20210126/SpecLib.hdf5"
  galaxy_SED: "Templates/Galaxy/"


###############################################
# Instrumental effects setting
# (NOTE) Here only used to construct
# ObservationSim.Instrument.Chip object
# (TODO) Should readout from header
###############################################
ins_effects:
  # switches
  bright_fatter:  ON # Whether to simulate Brighter-Fatter (also diffusion) effect

  # values
  # dark_exptime:   300   # Exposure time for dark current frames [seconds]
  # flat_exptime:   150   # Exposure time for flat-fielding frames [seconds]
  # readout_time:   40    # The read-out time for each channel [seconds]
  # df_strength:    2.3   # Sillicon sensor diffusion strength
  # bias_level:     500   # bias level [e-/pixel]
  # gain:           1.1   # Gain
  # full_well:      90000 # Full well depth [e-]

###############################################
# Random seeds
###############################################
random_seeds:
  seed_Av:              121212    # Seed for generating random intrinsic extinction

###############################################
# Measurement setting
###############################################
measurement_setting:
  input_img_list: "/share/home/fangyuedong/injection_pipeline/L1_INJECTED_20231203.list"
  # input_img_list: "/share/home/fangyuedong/injection_pipeline/input_L1_img_MSC_0000000.list"
  input_wht_list: "/share/home/fangyuedong/injection_pipeline/L1_WHT_20231203.list"
  input_flg_list: "/share/home/fangyuedong/injection_pipeline/L1_FLG_20231203.list"
  # input_psf_list: "/share/home/fangyuedong/injection_pipeline/psf_img_MSC_0000000.list"
  sex_config: "/share/home/fangyuedong/injection_pipeline/config/default.config"
  sex_param: "/share/home/fangyuedong/injection_pipeline/config/default.param"
  n_jobs: 18
  output_dir: "/share/home/fangyuedong/injection_pipeline/workspace"
 No newline at end of file
+100 −27
Original line number Diff line number Diff line
@@ -8,16 +8,30 @@ from ObservationSim.Instrument import Telescope, Filter, FilterParam

VC_A = 2.99792458e+18  # speed of light: A/s

# def define_options():
#     parser = argparse.ArgumentParser()
#     parser.add_argument('--TU_catalog', dest='TU_catalog', type=str, required=True,
#                         help='path to the (injected) truth catalog')
#     parser.add_argument('--source_catalog', dest='source_catalog', type=str, required=True,
#                         help='path to the (extracted) injected catalog')
#     parser.add_argument('--orig_catalog', dest='orig_catalog', type=str, required=True,
#                         help='path to the (extracted) original catalog')
#     parser.add_argument('--image', dest='image', type=str, required=True,
#                         help='path to the image, used to get the header info')
#     parser.add_argument('--output_dir', dest='output_dir', type=str, required=False,
#                         default='./workspace', help='output path')
#     return parser

def define_options():
    parser = argparse.ArgumentParser()
    parser.add_argument('--TU_catalog', dest='TU_catalog', type=str, required=True,
                        help='path to the (injected) truth catalog')
    parser.add_argument('--source_catalog', dest='source_catalog', type=str, required=True,
    parser.add_argument('--TU_catalog_list', dest='TU_catalog_list', type=str, required=True,
                        help='path to the list of (injected) truth catalog')
    parser.add_argument('--source_catalog_list', dest='source_catalog_list', type=str, required=True,
                        help='path to the (extracted) injected catalog')
    parser.add_argument('--orig_catalog', dest='orig_catalog', type=str, required=True,
                        help='path to the (extracted) original catalog')
    parser.add_argument('--image', dest='image', type=str, required=True,
                        help='path to the image, used to get the header info')
    # parser.add_argument('--orig_catalog', dest='orig_catalog', type=str, required=True,
    #                     help='path to the list of (extracted) original catalog')
    # parser.add_argument('--image', dest='image', type=str, required=True,
    #                     help='path to the image, used to get the header info')
    parser.add_argument('--output_dir', dest='output_dir', type=str, required=False,
                        default='./workspace', help='output path')
    return parser
@@ -72,13 +86,24 @@ def convert_catalog(catname):
    fits_filename = os.path.join(data_dir, base_name + '.fits')
    text_file.write(fits_filename, overwrite=True)

def validation_hist(val, idx, name="val", nbins=10, fig_name='detected_counts.png', output_dir='./'):
def validation_hist(val, idx, name="val", nbins=10, bins=None, fig_name='detected_counts.png', output_dir='./', create_figure=True):
    if bins is None:
        counts, bins = np.histogram(val, bins=nbins)
    else:
        counts, bins = np.histogram(val, bins=bins)
    is_empty = np.full(len(val), False)
    for i in range(len(idx)):
        if idx[i].size == 0:
            is_empty[i] = True
    if bins is None:
        counts_detected, _ = np.histogram(val[~is_empty], bins=nbins)
    else:
        counts_detected, _ = np.histogram(val[~is_empty], bins=bins)
    if create_figure:
        create_hist_figure(counts, counts_detected, bins, name, output_dir, fig_name)
    return counts, counts_detected, bins

def create_hist_figure(counts, counts_detected, bins, name="val", output_dir='./', fig_name='detected_counts.png'):
    plt.figure()
    plt.stairs(counts, bins, color='r', label='TU objects')
    plt.stairs(counts_detected, bins, color='g', label='Detected')
@@ -87,22 +112,38 @@ def validation_hist(val, idx, name="val", nbins=10, fig_name='detected_counts.pn
    plt.legend(loc='upper right', fancybox=True)
    fig_name = os.path.join(output_dir, fig_name)
    plt.savefig(fig_name)
    return counts, bins

def hist_fraction(val, idx, name='val', nbins=10, normed=False, output_dir='./'):
def hist_fraction(val, idx, name='val', nbins=10, bins=None, output_dir='./', fig_name="completeness_fraction.png"):
    if bins is None:
        counts, bins = np.histogram(val, bins=nbins)
    else:
        counts, bins = np.histogram(val, bins=bins)
    is_empty = np.full(len(val), False)
    for i in range(len(idx)):
        if idx[i].size == 0:
            is_empty[i] = True
    counts_detected, _ = np.histogram(val[~is_empty], bins=nbins, density=normed)
    if bins is None:
        counts_detected, _ = np.histogram(val[~is_empty], bins=nbins)
    else:
        counts_detected, _ = np.histogram(val[~is_empty], bins=bins)
    fraction = counts_detected / counts
    fraction[np.where(np.isnan(fraction))[0]] = 0.
    plt.figure()
    plt.stairs(fraction, bins, color='r', label='completeness fraction')
    plt.xlabel(name, size='x-large')
    plt.title("Completeness Fraction")
    fig_name = os.path.join(output_dir, "completeness_fraction_%s.png"%(name))
    fig_name = os.path.join(output_dir, fig_name)
    plt.savefig(fig_name)
    return fraction

def create_fraction_figure(counts, counts_detected, bins, name='val', output_dir='./', fig_name="completeness_fraction.png"):
    fraction = counts_detected / counts
    fraction[np.where(np.isnan(fraction))[0]] = 0.
    plt.figure()
    plt.stairs(fraction, bins, color='r', label='completeness fraction')
    plt.xlabel(name, size='x-large')
    plt.title("Completeness Fraction")
    fig_name = os.path.join(output_dir, fig_name)
    plt.savefig(fig_name)
    return fraction

@@ -116,6 +157,25 @@ def calculate_fraction(TU_catalog, source_catalog, output_dir, nbins=10):
    fraction = hist_fraction(val=mag_TU, idx=idx1, name="mag_injected", nbins=10, output_dir=output_dir)
    return counts, bins, fraction

def calculate_fraction_multi_cats(TU_catalog_list, source_catalog_list, output_dir, nbins=10):
    counts = np.zeros(nbins)
    counts_detected = np.zeros(nbins)
    bins = np.linspace(18, 26, num=(nbins+1))
    for i in range(len(TU_catalog_list)):
        TU_catalog = TU_catalog_list[i]
        source_catalog = source_catalog_list[i]
        convert_catalog(TU_catalog)
        x_TU_temp, y_TU_temp, col_list = read_catalog(TU_catalog + '.fits', ext_num=1, ra_name="xImage", dec_name="yImage", col_list=["mag"])
        mag_TU_temp = col_list[0]
        x_source_temp, y_source_temp, _ = read_catalog(source_catalog, ext_num=1, ra_name="X_IMAGE", dec_name="Y_IMAGE")
        idx1, _, = match_catalogs_img(x1=x_TU_temp, y1=y_TU_temp, x2=x_source_temp, y2=y_source_temp)
        counts_temp, counts_detected_temp, _ = validation_hist(val=mag_TU_temp, idx=idx1, name="mag_injected", bins=bins, output_dir=output_dir, create_figure=False)
        counts += counts_temp
        counts_detected += counts_detected_temp
    create_hist_figure(counts, counts_detected, bins, "mag_injected", output_dir)
    fraction = create_fraction_figure(counts, counts_detected, bins, 'mag_injected', output_dir)
    return counts, counts_detected, bins, fraction

def calculate_undetected_flux(orig_cat, mag_bins, fraction, mag_low=20.0, mag_high=26.0, image=None,  output_dir='./'):
    # Get info from original image
    hdu = fits.open(image)
@@ -166,19 +226,32 @@ def calculate_undetected_flux(orig_cat, mag_bins, fraction, mag_low=20.0, mag_hi
    undetected_flux /= (float(nx_pix) * float(ny_pix))
    return undetected_flux

# if __name__ == "__main__":
#     args = define_options().parse_args()
#     counts, bins, fraction = calculate_fraction(
#         TU_catalog=args.TU_catalog,
#         source_catalog=args.source_catalog,
#         output_dir=args.output_dir,
#         nbins=20
#     )
#     undetected_flux = calculate_undetected_flux(
#         orig_cat=args.orig_catalog,
#         mag_bins=bins,
#         fraction=fraction,
#         image=args.image,
#         output_dir=args.output_dir,
#     )
#     print(undetected_flux)

if __name__ == "__main__":
    args = define_options().parse_args()
    counts, bins, fraction = calculate_fraction(
        TU_catalog=args.TU_catalog,
        source_catalog=args.source_catalog,
    with open(args.TU_catalog_list) as file:
        TU_catalog_list = [line.rstrip() for line in file]
    with open(args.source_catalog_list) as file:
        source_catalog_list = [line.rstrip() for line in file]
    counts, counts_detected, bins, fraction = calculate_fraction_multi_cats(
        TU_catalog_list=TU_catalog_list,
        source_catalog_list=source_catalog_list,
        output_dir=args.output_dir,
        nbins=20
    )
 No newline at end of file
    undetected_flux = calculate_undetected_flux(
        orig_cat=args.orig_catalog,
        mag_bins=bins,
        fraction=fraction,
        image=args.image,
        output_dir=args.output_dir,
    )
    print(undetected_flux)
 No newline at end of file
+5 −19
Original line number Diff line number Diff line
import numpy as np
import astropy.units as u
import matplotlib.pyplot as plt
from astropy.coordinates import SkyCoord
from astropy.io import fits
from astropy.io import ascii
from sklearn.neighbors import BallTree

# TU_catalog = "test_RectGrid_20220628.cat"
# source_catalog = "extracted_test_RectGrid_20220628.fits"
TU_catalog = "injected_bkgsub_img.cat"
source_catalog = "extracted_injected_bkgsub_img.fits"

@@ -26,7 +23,6 @@ def read_catalog(catname, ext_num=1, ra_name='ra', dec_name='dec', col_list=[]):
    if len(col_list) > 0:
        for col in col_list:
            col_other.append(data[col])
    # print(ra, dec)
    return ra, dec, col_other

def match_catalogs_sky(ra1, dec1, ra2, dec2, max_dist=0.6, others1=[], others2=[], thresh=[]):
@@ -37,33 +33,24 @@ def match_catalogs_sky(ra1, dec1, ra2, dec2, max_dist=0.6, others1=[], others2=[
    # print(idx2)
    # print(np.shape(idx1))
    # print(np.shape(idx2))
    # TODO

def match_catalogs_img(x1, y1, x2, y2, max_dist=0.5, others1=[], others2=[], thresh=[]):
def match_catalogs_img(x1, y1, x2, y2, max_dist=2, others1=[], others2=[], thresh=[]):
    cat1 = np.array([(x, y) for x,y in zip(x1, y1)])
    cat2 = np.array([(x, y) for x,y in zip(x2, y2)])
    # print(np.shape(cat1))
    # print(np.shape(cat2))
    tree = BallTree(cat2)
    idx1 = tree.query_radius(cat1, r = max_dist)
    tree = BallTree(cat1)
    idx2 = tree.query_radius(cat2, r = max_dist)
    # print(np.shape(idx1))
    tot = 0
    print(idx1)
    for idx in idx1:
        if len(idx) == 0:
            continue
        if len(idx) > 1: print(len(idx))
        tot += 1
    print(tot)
    print("number of matched sources = ", tot)
    return idx1, idx2

def validation_hist(val1, idx1, name="val1", nbins=10):
    counts, bins = np.histogram(val1)
    plt.stairs(counts, bins)
    plt.xlabel(name, size='x-large')
    plt.savefig("detection_completeness.png")
    # plt.show()

if __name__=="__main__":
    convert_catalog(TU_catalog)
    ra_TU, dec_TU, _ = read_catalog('test_ascii_to_fits.fits', ext_num=1, ra_name="ra", dec_name="dec")
@@ -74,4 +61,3 @@ if __name__=="__main__":
    # match_catalogs_sky(ra1=ra_TU, dec1=dec_TU, ra2=ra_source, dec2=dec_source)
    idx1, idx2, = match_catalogs_img(x1=x_TU, y1=y_TU, x2=x_source, y2=y_source)
    # print(ra_TU, dec_TU)
 No newline at end of file
    validation_hist(mag_TU, idx1, name="mag_injected")
 No newline at end of file
+232 −0
Original line number Diff line number Diff line
import os
import galsim
import random
import numpy as np
import h5py as h5
import healpy as hp
import astropy.constants as cons
import traceback
from astropy.coordinates import spherical_to_cartesian
from astropy.table import Table
from scipy import interpolate
from datetime import datetime

from ObservationSim.MockObject import CatalogBase, Star, Galaxy, Quasar
from ObservationSim.MockObject._util import tag_sed, getObservedSED, getABMAG, integrate_sed_bandpass, comoving_dist
from ObservationSim.Astrometry.Astrometry_util import on_orbit_obs_position

# (TEST)
from astropy.cosmology import FlatLambdaCDM
from astropy import constants
from astropy import units as U

try:
    import importlib.resources as pkg_resources
except ImportError:
    # Try backported to PY<37 'importlib_resources'
    import importlib_resources as pkg_resources

# CONSTANTS
NSIDE = 128

def get_bundleIndex(healpixID_ring, bundleOrder=4, healpixOrder=7):
    assert NSIDE == 2**healpixOrder
    shift = healpixOrder - bundleOrder
    shift = 2*shift

    nside_bundle = 2**bundleOrder
    nside_healpix= 2**healpixOrder

    healpixID_nest= hp.ring2nest(nside_healpix, healpixID_ring)
    bundleID_nest = (healpixID_nest >> shift)
    bundleID_ring = hp.nest2ring(nside_bundle, bundleID_nest)

    return bundleID_ring

class SimCat(CatalogBase):
    def __init__(self, config, chip, nobjects=None):
        super().__init__()
        self.cat_dir = os.path.join(config["data_dir"], config["catalog_options"]["input_path"]["cat_dir"])
        self.seed_Av = config["catalog_options"]["seed_Av"]
        self.cosmo = FlatLambdaCDM(H0=67.66, Om0=0.3111)

        with pkg_resources.path('Catalog.data', 'SLOAN_SDSS.g.fits') as filter_path:
            self.normF_star = Table.read(str(filter_path))

        self.config = config
        self.chip = chip

        galaxy_dir = config["catalog_options"]["input_path"]["galaxy_cat"]
        self.galaxy_path = os.path.join(self.cat_dir, galaxy_dir)
        self.galaxy_SED_path = os.path.join(config["data_dir"], config["catalog_options"]["SED_templates_path"]["galaxy_SED"])
        self._load_SED_lib_gals()

        if "rotateEll" in config["catalog_options"]:
            self.rotation = float(int(config["catalog_options"]["rotateEll"]/45.))
        else:
            self.rotation = 0.
        
        self._get_healpix_list()
        self._load(nobjects=nobjects)

    def _get_healpix_list(self):
        self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2)
        ra_min, ra_max, dec_min, dec_max = self.sky_coverage.xmin, self.sky_coverage.xmax, self.sky_coverage.ymin, self.sky_coverage.ymax
        ra = np.deg2rad(np.array([ra_min, ra_max, ra_max, ra_min]))
        dec = np.deg2rad(np.array([dec_max, dec_max, dec_min, dec_min]))
        self.pix_list = hp.query_polygon(
            NSIDE,
            hp.ang2vec(np.radians(90.) - dec, ra),
            inclusive=True
        )
        if self.logger is not None:
            msg = str(("HEALPix List: ", self.pix_list))
            self.logger.info(msg)
        else:
            print("HEALPix List: ", self.pix_list)

    def load_norm_filt(self, obj):
        if obj.type == "star":
            return self.normF_star
        elif obj.type == "galaxy" or obj.type == "quasar":
            return None
        else:
            return None

    def _load_SED_lib_gals(self):
        pcs = h5.File(os.path.join(self.galaxy_SED_path, "pcs.h5"), "r")
        lamb = h5.File(os.path.join(self.galaxy_SED_path, "lamb.h5"), "r")
        self.lamb_gal = lamb['lamb'][()]
        self.pcs = pcs['pcs'][()]

    def _load_gals(self, gals, pix_id=None, cat_id=0, nobjects=1):
        # Load how mnay objects?
        max_ngals = len(gals['ra'])
        remain = nobjects 
        for igals in range(max_ngals):
            if remain == 0:
                break
            param = self.initialize_param()
            param['ra'] = ra_arr[igals]
            param['dec'] = dec_arr[igals]
            param['ra_orig'] = gals['ra'][igals]
            param['dec_orig'] = gals['dec'][igals]
            # [TODO]
            param['mag_use_normal'] = gals['mag_csst_%s'%(self.filt.filter_type)][igals]
            # if self.filt.is_too_dim(mag=param['mag_use_normal'], margin=self.config["obs_setting"]["mag_lim_margin"]):
            #     continue

            param['z'] = gals['redshift'][igals]
            param['model_tag'] = 'None'
            param['g1'] = gals['shear'][igals][0]
            param['g2'] = gals['shear'][igals][1]
            param['kappa'] = gals['kappa'][igals]
            param['e1'] = gals['ellipticity_true'][igals][0]
            param['e2'] = gals['ellipticity_true'][igals][1]
            
            # For shape calculation
            
            param['ell_total'] = np.sqrt(param['e1']**2 + param['e2']**2)
            if param['ell_total'] > 0.9:
                continue
            
            remain -= 1
            param['e1_disk'] = param['e1']
            param['e2_disk'] = param['e2']
            param['e1_bulge'] = param['e1']
            param['e2_bulge'] = param['e2']


            param['delta_ra'] = 0
            param['delta_dec'] = 0

            # Masses
            param['bulgemass'] = gals['bulgemass'][igals]
            param['diskmass'] = gals['diskmass'][igals]

            param['size'] = gals['size'][igals]
            if param['size'] > self.max_size:
                self.max_size = param['size']

            # Sersic index
            param['disk_sersic_idx'] = 1.
            param['bulge_sersic_idx'] = 4.

            # Sizes
            param['bfrac'] = param['bulgemass']/(param['bulgemass'] + param['diskmass'])
            if param['bfrac'] >= 0.6:
                param['hlr_bulge'] = param['size']
                param['hlr_disk'] = param['size'] * (1. - param['bfrac'])
            else:
                param['hlr_disk'] = param['size']
                param['hlr_bulge'] = param['size'] * param['bfrac']

            # SED coefficients
            param['coeff'] = gals['coeff'][igals]
            param['detA'] = gals['detA'][igals]

            # Others
            param['galType'] = gals['type'][igals]
            param['veldisp'] = gals['veldisp'][igals]
            
            # TEST no redening and no extinction
            param['av'] = 0.0
            param['redden'] = 0

            param['star'] = 0   # Galaxy

            # TEMP
            self.ids += 1
            # param['id'] = self.ids
            param['id'] = '%06d'%(int(pix_id)) + '%06d'%(cat_id) + '%08d'%(igals)
            
            if param['star'] == 0:
                obj = Galaxy(param, self.rotation, logger=self.logger)

            self.objs.append(obj)
        return remain
    
    def _load(self, nobjects=1):
        from itertools import cycle
        self.objs = []
        self.ids = 0
        to_be_read_in = nobjects
        pool = cycle(self.pix_list)
        for pix in pool:
            try:
                if to_be_read_in == 0:
                    break
                bundleID  = get_bundleIndex(pix)
                file_path = os.path.join(self.galaxy_path, "galaxies_C6_bundle{:06}.h5".format(bundleID))
                gals_cat = h5.File(file_path, 'r')['galaxies']
                gals = gals_cat[str(pix)]
                to_be_read_in = self._load_gals(gals, pix_id=pix, cat_id=bundleID, n_objects=to_be_read_in)
                del gals
            except Exception as e:
                traceback.print_exc()
                print(e)

    def load_sed(self, obj, **kwargs):
        factor = 10**(-.4 * self.cosmo.distmod(obj.z).value)
        flux = np.matmul(self.pcs, obj.coeff) * factor
        #  if np.any(flux < 0):
        #     raise ValueError("Glaxy %s: negative SED fluxes"%obj.id)
        flux[flux < 0] = 0.
        sedcat = np.vstack((self.lamb_gal, flux)).T
        sed_data = getObservedSED(
            sedCat=sedcat,
            redshift=obj.z,
            av=obj.param["av"],
            redden=obj.param["redden"]
        )
        wave, flux = sed_data[0], sed_data[1]
        speci = interpolate.interp1d(wave, flux)
        lamb = np.arange(2000, 11001+0.5, 0.5)
        y = speci(lamb)
        # erg/s/cm2/A --> photon/s/m2/A
        all_sed = y * lamb / (cons.h.value * cons.c.value) * 1e-13
        sed = Table(np.array([lamb, all_sed]).T, names=('WAVELENGTH', 'FLUX'))
        
        del wave
        del flux
        return sed
 No newline at end of file
Loading