Skip to content
GitLab
Commit 1f233449 authored by Fang Yuedong's avatar Fang Yuedong
Browse files

add entry points in setup.py to make executables

parent 72c543f5
Show whitespace changes
Inline Side-by-side
README.md
View file @ 1f233449
......@@ -13,7 +13,7 @@
* This step removes all the instrumental related effects as well as cosmoic rays
* modify the corresponding ```config_detrending.yaml``` file to configure all running options
```
python ./measurement_pipeline/run_csst_msc_instrument.py /path/to/config_detrending.yaml
run_csst_detrending /path/to/config_detrending.yaml
```
* Products:
* L1 calibrated images (detrended + cosmic rays removed)
......@@ -25,7 +25,7 @@
* Modify the corresponding ```config_injection.yaml``` file to configure all running options
* An object catalog class which inherits from ```observation_sim.mock_objects.CatalogBase``` needs to be implemented and imported in ```injection.injection_pipeline.py```. This is to construct a list of to be injected ```observation_sim.mock_objects.MockObject```.
```
python ./injection_pipeline/injection_pipeline.py /path/to/config_injection.yaml
run_csst_injection /path/to/config_injection.yaml
```
* Products:
* L1 calibrated images with synthetic sources injected
......@@ -37,7 +37,7 @@
* Re-detect and redo photometry measurements on the injected images
* modify the corresponding ```config_photometry.yaml``` file to configure all running options
```
python ./measurement_pipeline/run_csst_msc_mbi_photometry.py /path/to/config_photometry.yaml
run_csst_photometry /path/to/config_photometry.yaml
```
* Products:
* Detection catalogs with photometric information
......
config/config_detrending.yaml
View file @ 1f233449
input_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W2/"
output_dir: "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/"
calib_data_path: "/public/home/fangyuedong/project/calib_data/"
pointing_label_list: ["MSC_0000000"]
chip_label_list: ["07"]
\ No newline at end of file
config/config_injection.yaml
View file @ 1f233449
......@@ -20,7 +20,7 @@ pos_sampling:
# object_density: 37 # arcmin^-2
input_dir: "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W2/"
output_dir: "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/"
output_dir: "/public/home/fangyuedong/project/demo_csst_injection/test/"
pointing_label_list: ["MSC_0000000"]
chip_label_list: ["07"]
......
config/config_photometry.yaml
View file @ 1f233449
input_dir: "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/"
# input_dir: "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/"
input_dir: "/public/home/fangyuedong/project/demo_csst_injection/test/"
flag_weight_dir: "/public/home/fangyuedong/project/demo_csst_injection/L1_products/50sqDeg_Photo_W2/"
output_dir: "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/"
output_dir: "/public/home/fangyuedong/project/demo_csst_injection/test/"
pointing_label_list: ["MSC_0000000"]
chip_label_list: ["07"]
calib_data_path: "/public/home/fangyuedong/project/calib_data/"
\ No newline at end of file
evaluation/evaluation_example.ipynb
View file @ 1f233449
%% Cell type:code id:dc2c5d8e tags:
%% Cell type:code id:809b5d07 tags:
 
``` python
import os
import warnings
import importlib
import numpy as np
import matplotlib.pyplot as plt
from pathlib import Path
from glob import glob
from astropy.io import ascii, fits
from astropy.utils.exceptions import AstropyWarning
warnings.simplefilter('ignore', category=AstropyWarning)
 
import evaluation.evaluation_utils as evaluation_utils
from evaluation.calculate_completeness_fraction import validation_hist
from evaluation.cross_match_catalogs import read_catalog, match_catalogs_img
 
importlib.reload(evaluation_utils)
%matplotlib inline
```
 
%% Cell type:code id:92d0ff48 tags:
%% Cell type:code id:212a642a tags:
 
``` python
chip_label = "07"
pointing_label = "MSC_0000000"
 
fig_root_dir = "./fig_for_demonstration"
if not os.path.exists(fig_root_dir):
os.makedirs(fig_root_dir)
 
l1_img_dir = "/public/home/fangyuedong/project/demo_csst_injection/L1_products/50sqDeg_Photo_W2/%s/"%(pointing_label)
l1_injected_dir = "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/%s"%(pointing_label)
```
 
%% Cell type:markdown id:1d59e962 tags:
%% Cell type:markdown id:45201a01 tags:
 
# Visualize the injection
Use evaluation.evaluation_utils.plot_injection_comparison to plot before vs after injection, and the difference
 
%% Cell type:code id:74f801a0 tags:
%% Cell type:code id:230134ea tags:
 
``` python
l1_img = glob(l1_img_dir + "/*_%s_img*.fits"%(chip_label))[0]
l1_flg = glob(l1_img_dir + "/*_%s_flg*.fits"%(chip_label))[0]
injected_img = glob(l1_injected_dir + "/*_%s_img_*.fits"%(chip_label))[0]
 
evaluation_utils.plot_injection_comparison(orig_img=l1_img,
injected_img=injected_img,
flg_img=l1_flg,
save_fig_dir=fig_root_dir,
fig_prefix="%s_%s_%s_"%("demo", pointing_label, chip_label),
figsize=(8, 8))
```
 
%% Output
 
 
 
 
%% Cell type:markdown id:1202b0bb tags:
%% Cell type:markdown id:a97854f7 tags:
 
# Matching catalogs and check detection completeness
* For each pair of ground truth and detected catalogs from an injected image, use ```evaluation.cross_match_catalogs.match_catalogs_img``` to match the source by image coordinate
* Use ```evaluation.evaluation_utils.validation_hist``` to make a histogram for each pair. Accumulate the counts
* Use ```evaluation.evaluation_utils.create_fraction_figure``` to generate the final detection fraction histogram
 
%% Cell type:code id:6ebebcef tags:
%% Cell type:code id:508205b5 tags:
 
``` python
injected_truth_dir = "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/"
injected_l1_output_dir = "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/"
 
nbins = 50
bins = np.linspace(16., 28., nbins+1)
 
counts = np.zeros(nbins)
counts_detected = np.zeros(nbins)
 
cat_path_list_injected_truth = glob(injected_truth_dir + "*/*_%s_*.cat*"%(chip_label))
cat_path_list_injected_l1 = glob(injected_l1_output_dir + "*/*%s_cat*"%(chip_label))
 
for cat_path_injected_truth in cat_path_list_injected_truth:
print("Injected truth catalog: ", os.path.basename(cat_path_injected_truth))
obs_id = cat_path_injected_truth.split('/')[-2]
data = ascii.read(cat_path_injected_truth)
x_truth = data["xImage"]
y_truth = data["yImage"]
mag_truth = data["mag"]
 
cat_path_injected_l1 = os.path.join(injected_l1_output_dir,
obs_id,
os.path.basename(cat_path_injected_truth).replace("img", "cat").replace(".cat", ".fits"))
print("L1 processed photometry catalog: ", os.path.basename(cat_path_injected_l1))
hdu = fits.open(cat_path_injected_l1)
x_measure = hdu[1].data["X"]
y_measure = hdu[1].data["Y"]
mag_measure = hdu[1].data["Mag_Kron"]
 
idx1, _, = match_catalogs_img(x1=x_truth, y1=y_truth, x2=x_measure, y2=y_measure)
 
counts_temp, counts_detected_temp, _ = evaluation_utils.validation_hist(val=mag_truth,
idx=idx1,
name="mag_injected",
bins=bins,
output_dir='./',
create_figure=False)
counts += counts_temp
counts_detected += counts_detected_temp
 
fig1, ax1 = evaluation_utils.create_hist_figure(counts, counts_detected, bins, "mag_injected", './')
fig1.savefig(os.path.join(fig_root_dir, "detection_recovered_W2.png"))
fig1
fig2, ax2, fraction = evaluation_utils.create_fraction_figure(counts, counts_detected, bins, 'mag_injected', './', label="W2")
```
 
%% Output
 
Injected truth catalog: CSST_MSC_MS_SCIE_20220708044845_20220708045115_10160000000_07_img_L1_injected.cat
L1 processed photometry catalog: CSST_MSC_MS_SCIE_20220708044845_20220708045115_10160000000_07_cat_L1_injected.fits
number of matched sources = 389
 
/public/home/fangyuedong/.local/lib/python3.11/site-packages/csst_source_injection-1.0.0-py3.11.egg/evaluation/evaluation_utils.py:109: RuntimeWarning: invalid value encountered in divide
fraction = counts_detected / counts
 
 
 
%% Cell type:markdown id:13b2c150 tags:
%% Cell type:markdown id:0eb991ca tags:
 
# Evaluate recovered photometry
* Use ```evaluation.evaluation_utils.plot_mag_comparison``` to make diff(mag) vs true_mag plot
 
%% Cell type:code id:d4738454 tags:
%% Cell type:code id:2c5648aa tags:
 
``` python
importlib.reload(evaluation_utils)
injected_truth_dir = "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/"
injected_l1_output_dir = "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/"
 
cat_path_list_injected_truth = glob(injected_truth_dir + "*/*_%s_*.cat*"%(chip_label))
evaluation_utils.plot_mag_comparison(truth_cat_list=cat_path_list_injected_truth,
measured_cat_root_dir=injected_l1_output_dir,
save_fig_dir=fig_root_dir,
fig_prefix="%s_%s_"%("demo", chip_label),
title="%s, chip: %s"%("demo", chip_label))
```
 
%% Output
 
Injected truth catalog: CSST_MSC_MS_SCIE_20220708044845_20220708045115_10160000000_07_img_L1_injected.cat
L1 processed photometry catalog: CSST_MSC_MS_SCIE_20220708044845_20220708045115_10160000000_07_img_L1_injected.cat
number of matched sources = 389
 
injection_pipeline/InjectionCatalog.py
View file @ 1f233449
import numpy as np
import galsim
import Grid
import mathutil as util
import injection_pipeline.Grid as Grid
import injection_pipeline.mathutil as util
class InjectionCatalog(object):
......
injection_pipeline/Injector.py
View file @ 1f233449
......@@ -4,7 +4,7 @@ import logging
import os
import numpy as np
import grid
import injection_pipeline.Grid as grid
class AddOnImageBuilder(galsim.config.image_scattered.ScatteredImageBuilder):
......
injection_pipeline/injection_pipeline.py
View file @ 1f233449
......@@ -3,10 +3,10 @@ import os
import yaml
from glob import glob
from SingleEpochImage import SingleEpochImage
from InjectionCatalog import InjectionCatalog
from Catalog.C6_SimCat import SimCat
from config import parse_args
from injection_pipeline.SingleEpochImage import SingleEpochImage
from injection_pipeline.InjectionCatalog import InjectionCatalog
from injection_pipeline.Catalog.C6_SimCat import SimCat
from injection_pipeline.config import parse_args
class InjectionPipeline(object):
......@@ -118,6 +118,10 @@ class InjectionPipeline(object):
image.inject_objects(pos=inject_cat.pos, cat=input_cat)
image.save_injected_img()
def main():
pipeline = InjectionPipeline()
pipeline.run_pointing_list(Catalog=SimCat)
if __name__ == "__main__":
# input_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W2/"
......@@ -130,11 +134,6 @@ if __name__ == "__main__":
# output_dir = "/public/home/fangyuedong/project/test_photometry/50sqDeg_Photo_W2/"
# config_file = "/public/home/fangyuedong/project/injection_pipeline/config/config_injection.yaml"
# pipeline = InjectionPipeline(config_file=config_file)
pipeline = InjectionPipeline()
# pipeline.run_pointing_list(input_dir=input_dir,
# pointing_label_list=pointing_label_list,
# output_dir=output_dir,
# chip_label_list=chip_label_list,
# Catalog=SimCat)
pipeline.run_pointing_list(Catalog=SimCat)
# pipeline = InjectionPipeline()
# pipeline.run_pointing_list(Catalog=SimCat)
main()
measurement_pipeline/L1_pipeline/csst_msc_instrument/csst_msc_mbi_instrument.py
View file @ 1f233449
......@@ -23,7 +23,7 @@ from astropy.stats import sigma_clipped_stats
# subtract_dark,
# FlagConfig,
# )
from L1_pipeline.csst_msc_instrument.image import (
from measurement_pipeline.L1_pipeline.csst_msc_instrument.image import (
remove_cr_deepcr,
subtract_bias,
subtract_dark,
......
measurement_pipeline/L1_pipeline/csst_msc_instrument/image/cosmicray/deepcr.py
View file @ 1f233449
......@@ -14,12 +14,14 @@ import os
import numpy as np
import deepCR
from ..config import FlagConfig
from measurement_pipeline.L1_pipeline.csst_msc_instrument.image.config import FlagConfig
import importlib.resources as pkg_resources
__all__ = ["remove_cr_deepcr", ]
MODEL_DIR = os.path.join(os.path.dirname(
os.path.abspath(__file__)), 'cr_model')
# MODEL_DIR = os.path.join(os.path.dirname(
# os.path.abspath(__file__)), 'cr_model')
DEEPCR_MODEL_PATH = {
'01': '2022-04-26_15_samples_32_layers_epoch30.pth',
......@@ -86,7 +88,9 @@ def remove_cr_deepcr(image: np.ndarray,
"""
inpaint = False
inpaint_model = 'ACS-WFC-F606W-2-32'
model_path = os.path.join(MODEL_DIR, DEEPCR_MODEL_PATH[chipid])
# model_path = os.path.join(MODEL_DIR, DEEPCR_MODEL_PATH[chipid])
with pkg_resources.path('measurement_pipeline.L1_pipeline.csst_msc_instrument.image.cosmicray.cr_model', DEEPCR_MODEL_PATH[chipid]) as p:
model_path = p
model = deepCR.deepCR(
model_path,
inpaint_model,
......
measurement_pipeline/L1_pipeline/csst_msc_instrument/image/overscan.py
View file @ 1f233449
......@@ -18,7 +18,7 @@ from scipy.special import erf
from scipy.stats import trim_mean, t, sigmaclip
from astropy.stats import mad_std, sigma_clip
from .config import get_array_config
from measurement_pipeline.L1_pipeline.csst_msc_instrument.image.config import get_array_config
__all__ = ["average_overscan", "smooth_overscan", "correct_overscan", ]
......
measurement_pipeline/L1_pipeline/csst_msc_mbi_photometry/csst_photometry.py
View file @ 1f233449
......@@ -75,8 +75,8 @@ from scipy.interpolate import UnivariateSpline
from astropy.coordinates import SkyCoord
from shutil import which
import astropy.units as u
from .stats import sigmaclip_limitsig, weighted_mean, closest_match
from .mag_flux_convert import fluxerr2magerr
from measurement_pipeline.L1_pipeline.csst_msc_mbi_photometry.stats import sigmaclip_limitsig, weighted_mean, closest_match
from measurement_pipeline.L1_pipeline.csst_msc_mbi_photometry.mag_flux_convert import fluxerr2magerr
# from csst_common import CsstResult, CsstStatus
from typing import Optional
......
measurement_pipeline/run_csst_msc_instrument.py
View file @ 1f233449
......@@ -5,8 +5,8 @@ import yaml
from glob import glob
import mpi4py.MPI as MPI
# from .L1_pipeline.csst_msc_instrument.csst_msc_mbi_instrument import core_msc_l1_mbi_instrument
from L1_pipeline.csst_msc_instrument.csst_msc_mbi_instrument import core_msc_l1_mbi_instrument
from config import parse_args
from measurement_pipeline.L1_pipeline.csst_msc_instrument.csst_msc_mbi_instrument import core_msc_l1_mbi_instrument
from measurement_pipeline.config import parse_args
def run_csst_msc_instrument(image_path,
......@@ -121,19 +121,7 @@ def run_pointing_list(input_dir,
calib_data_path=calib_data_path)
if __name__ == "__main__":
# input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W1/"
# pointing_label_list = ["MSC_0000000"]
# chip_label_list = ["08"]
# output_dir = "/public/home/fangyuedong/project/test_deepcr"
# input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W2/"
# output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W2/"
# input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W3/"
# output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/"
# pointing_label_list = ["MSC_0000000", "MSC_0000001",
# "MSC_0000002", "MSC_0000003", "MSC_0000004", "MSC_0000005", "MSC_0000006", "MSC_0000007", "MSC_0000008", "MSC_0000009"]
# chip_label_list = None
def main():
args = parse_args()
config_file = args.config_file
with open(config_file, "r") as stream:
......@@ -149,3 +137,19 @@ if __name__ == "__main__":
output_dir=config["output_dir"],
chip_label_list=config["chip_label_list"],
calib_data_path=config["calib_data_path"])
if __name__ == "__main__":
# input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W1/"
# pointing_label_list = ["MSC_0000000"]
# chip_label_list = ["08"]
# output_dir = "/public/home/fangyuedong/project/test_deepcr"
# input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W2/"
# output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W2/"
# input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W3/"
# output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/"
# pointing_label_list = ["MSC_0000000", "MSC_0000001",
# "MSC_0000002", "MSC_0000003", "MSC_0000004", "MSC_0000005", "MSC_0000006", "MSC_0000007", "MSC_0000008", "MSC_0000009"]
# chip_label_list = None
main()
measurement_pipeline/run_csst_msc_mbi_photometry.py
View file @ 1f233449
......@@ -4,9 +4,9 @@ import os
import yaml
from glob import glob
import mpi4py.MPI as MPI
from L1_pipeline.csst_msc_mbi_photometry.csst_photometry import core_msc_l1_mbi_phot
from measurement_pipeline.L1_pipeline.csst_msc_mbi_photometry.csst_photometry import core_msc_l1_mbi_phot
from typing import Optional
from config import parse_args
from measurement_pipeline.config import parse_args
def run_csst_msc_mbi_photometry(image_path,
output_dir,
......@@ -128,6 +128,23 @@ def run_pointing_list(input_dir,
output_dir=output_path,
flag_weight_dir=fw_dir)
def main():
args = parse_args()
config_file = args.config_file
with open(config_file, "r") as stream:
try:
config = yaml.safe_load(stream)
for key, value in config.items():
print(key + " : " + str(value))
except yaml.YAMLError as exc:
print(exc)
run_pointing_list(input_dir=config["input_dir"],
flag_weight_dir=config["flag_weight_dir"],
pointing_label_list=config["pointing_label_list"],
output_dir=config["output_dir"],
chip_label_list=config["chip_label_list"])
if __name__ == "__main__":
# input_dir = "/public/home/fangyuedong/project/injected_50sqDeg_L1_outputs"
......@@ -146,18 +163,4 @@ if __name__ == "__main__":
# # output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/"
# output_dir = "/public/home/fangyuedong/project/processed_injected_50sqDeg_L1_outputs/50sqDeg_Photo_W3/"
args = parse_args()
config_file = args.config_file
with open(config_file, "r") as stream:
try:
config = yaml.safe_load(stream)
for key, value in config.items():
print(key + " : " + str(value))
except yaml.YAMLError as exc:
print(exc)
run_pointing_list(input_dir=config["input_dir"],
flag_weight_dir=config["flag_weight_dir"],
pointing_label_list=config["pointing_label_list"],
output_dir=config["output_dir"],
chip_label_list=config["chip_label_list"])
main()
setup.py
View file @ 1f233449
......@@ -10,8 +10,18 @@ with open("requirements.txt", "r") as f:
setup(name='csst_source_injection',
version='1.0.0',
packages=find_packages(),
entry_points={
'console_scripts': [
'run_csst_injection=injection_pipeline.injection_pipeline:main',
'run_csst_detrending=measurement_pipeline.run_csst_msc_instrument:main',
'run_csst_photometry=measurement_pipeline.run_csst_msc_mbi_photometry:main'
]
},
package_data={
'injection_pipeline.Catalog.data': ["*.fits"],
'measurement_pipeline.L1_pipeline.csst_msc_mbi_photometry.data': ["*.config", "*.conv", "*.nnw", "*.param", "*.sex", "*.psfex"],
'measurement_pipeline.L1_pipeline.csst_msc_instrument': ["instrument_cor_demo.head"],
'measurement_pipeline.L1_pipeline.csst_msc_instrument.image.cosmicray.cr_model': ["*.pth"]
},
python_requires=">=3.11",
install_requires=requirements,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment