From 4b820c11ff77a5e6340793bec3688d04bac568e3 Mon Sep 17 00:00:00 2001 From: fangyuedong Date: Wed, 23 Oct 2024 04:33:15 +0800 Subject: [PATCH] add configuration files for detrending and photometry --- README.md | 17 +++++++- config/config_detrending.yaml | 4 ++ config/config_photometry.yaml | 6 +++ measurement_pipeline/config.py | 12 ++++++ .../run_csst_msc_instrument.py | 33 ++++++++++----- .../run_csst_msc_mbi_photometry.py | 42 ++++++++++++------- 6 files changed, 87 insertions(+), 27 deletions(-) create mode 100644 config/config_detrending.yaml create mode 100644 config/config_photometry.yaml create mode 100644 measurement_pipeline/config.py diff --git a/README.md b/README.md index c2e83d3..40e5ac1 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,22 @@ ``` ## Usage +### Run L1 Detrending pipeline (optional) +* If the L1 images are not available, we need first to run the CSST detrending pipeline on the raw images to get the L1 calibrated products. Otherwise this step can be ignored. +* This step removes all the instrumental related effects as well as cosmoic rays +* modify the corresponding ```config_detrending.yaml``` file to configure all running options +``` + python ./measurement_pipeline/run_csst_msc_instrument.py /path/to/config_detrending.yaml +``` ### Run Source Injection +* Inject souces onto the L1 (detrended) +* modify the corresponding ```config_injection.yaml``` file to configure all running options ``` python ./injection_pipeline/injection_pipeline.py /path/to/config_injection.yaml ``` -### Run L1 Detrending pipeline -### Run L1 Detection & Photometry pipeline \ No newline at end of file +### Run L1 Detection & Photometry pipeline +* Re-detect and redo the photometry measurements on the injected images +* modify the corresponding ```config_photometry.yaml``` file to configure all running options +``` + python ./measurement_pipeline/run_csst_msc_mbi_photometry.py /path/to/config_photometry.yaml +``` \ No newline at end of file diff --git a/config/config_detrending.yaml b/config/config_detrending.yaml new file mode 100644 index 0000000..9529741 --- /dev/null +++ b/config/config_detrending.yaml @@ -0,0 +1,4 @@ +input_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W2/" +output_dir: "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/" +pointing_label_list: ["MSC_0000000"] +chip_label_list: ["07"] \ No newline at end of file diff --git a/config/config_photometry.yaml b/config/config_photometry.yaml new file mode 100644 index 0000000..0da9688 --- /dev/null +++ b/config/config_photometry.yaml @@ -0,0 +1,6 @@ +input_dir: "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/" +flag_weight_dir: "/public/home/fangyuedong/project/demo_csst_injection/L1_products/50sqDeg_Photo_W2/" +output_dir: "/public/home/fangyuedong/project/demo_csst_injection/50sqDeg_Photo_W2/" +pointing_label_list: ["MSC_0000000"] +chip_label_list: ["07"] +calib_data_path: "/public/home/fangyuedong/project/calib_data/" \ No newline at end of file diff --git a/measurement_pipeline/config.py b/measurement_pipeline/config.py new file mode 100644 index 0000000..3c91869 --- /dev/null +++ b/measurement_pipeline/config.py @@ -0,0 +1,12 @@ +import argparse + +def parse_args(): + ''' + Parse command line arguments. Many of the following + can be set in the .yaml config file as well. + ''' + parser = argparse.ArgumentParser() + parser.add_argument('config_file', help='.yaml config file for injection settings.') + # parser.add_argument('-d', '--data_dir', help='Directory that houses the input data.') + # parser.add_argument('-w', '--work_dir', help='The path for output.') + return parser.parse_args() \ No newline at end of file diff --git a/measurement_pipeline/run_csst_msc_instrument.py b/measurement_pipeline/run_csst_msc_instrument.py index e0a9de5..e095ccf 100644 --- a/measurement_pipeline/run_csst_msc_instrument.py +++ b/measurement_pipeline/run_csst_msc_instrument.py @@ -1,10 +1,12 @@ """_summary_ """ import os +import yaml from glob import glob import mpi4py.MPI as MPI # from .L1_pipeline.csst_msc_instrument.csst_msc_mbi_instrument import core_msc_l1_mbi_instrument from L1_pipeline.csst_msc_instrument.csst_msc_mbi_instrument import core_msc_l1_mbi_instrument +from config import parse_args def run_csst_msc_instrument(image_path, @@ -126,13 +128,24 @@ if __name__ == "__main__": # output_dir = "/public/home/fangyuedong/project/test_deepcr" # input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W2/" # output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W2/" - input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W3/" - output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/" - pointing_label_list = ["MSC_0000000", "MSC_0000001", - "MSC_0000002", "MSC_0000003", "MSC_0000004", "MSC_0000005", "MSC_0000006", "MSC_0000007", "MSC_0000008", "MSC_0000009"] - chip_label_list = None - - run_pointing_list(input_dir=input_dir, - pointing_label_list=pointing_label_list, - output_dir=output_dir, - chip_label_list=chip_label_list) + # input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W3/" + # output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/" + # pointing_label_list = ["MSC_0000000", "MSC_0000001", + # "MSC_0000002", "MSC_0000003", "MSC_0000004", "MSC_0000005", "MSC_0000006", "MSC_0000007", "MSC_0000008", "MSC_0000009"] + # chip_label_list = None + + args = parse_args() + config_file = args.config_file + with open(config_file, "r") as stream: + try: + config = yaml.safe_load(stream) + for key, value in config.items(): + print(key + " : " + str(value)) + except yaml.YAMLError as exc: + print(exc) + + run_pointing_list(input_dir=config["input_dir"], + pointing_label_list=config["pointing_label_list"], + output_dir=config["output_dir"], + chip_label_list=config["chip_label_list"], + calib_data_path=config["calib_data_path"]) diff --git a/measurement_pipeline/run_csst_msc_mbi_photometry.py b/measurement_pipeline/run_csst_msc_mbi_photometry.py index eeb5a67..4ac8224 100644 --- a/measurement_pipeline/run_csst_msc_mbi_photometry.py +++ b/measurement_pipeline/run_csst_msc_mbi_photometry.py @@ -1,11 +1,12 @@ """_summary_ """ import os +import yaml from glob import glob import mpi4py.MPI as MPI from L1_pipeline.csst_msc_mbi_photometry.csst_photometry import core_msc_l1_mbi_phot from typing import Optional - +from config import parse_args def run_csst_msc_mbi_photometry(image_path, output_dir, @@ -135,17 +136,28 @@ if __name__ == "__main__": # chip_label_list = None # output_dir = "/public/home/fangyuedong/project/processed_injected_50sqDeg_L1_outputs" # flag_weight_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W2/" - flag_weight_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/" - - # input_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/" - input_dir = "/public/home/fangyuedong/project/injected_50sqDeg_L1_outputs/50sqDeg_Photo_W3/" - pointing_label_list = ["MSC_0000000", "MSC_0000001", - "MSC_0000002", "MSC_0000003", "MSC_0000004", "MSC_0000005", "MSC_0000006", "MSC_0000007", "MSC_0000008", "MSC_0000009"] - chip_label_list = None - # output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/" - output_dir = "/public/home/fangyuedong/project/processed_injected_50sqDeg_L1_outputs/50sqDeg_Photo_W3/" - run_pointing_list(input_dir=input_dir, - flag_weight_dir=flag_weight_dir, - pointing_label_list=pointing_label_list, - output_dir=output_dir, - chip_label_list=chip_label_list) + # flag_weight_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/" + + # # input_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/" + # input_dir = "/public/home/fangyuedong/project/injected_50sqDeg_L1_outputs/50sqDeg_Photo_W3/" + # pointing_label_list = ["MSC_0000000", "MSC_0000001", + # "MSC_0000002", "MSC_0000003", "MSC_0000004", "MSC_0000005", "MSC_0000006", "MSC_0000007", "MSC_0000008", "MSC_0000009"] + # chip_label_list = None + # # output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W3/" + # output_dir = "/public/home/fangyuedong/project/processed_injected_50sqDeg_L1_outputs/50sqDeg_Photo_W3/" + + args = parse_args() + config_file = args.config_file + with open(config_file, "r") as stream: + try: + config = yaml.safe_load(stream) + for key, value in config.items(): + print(key + " : " + str(value)) + except yaml.YAMLError as exc: + print(exc) + + run_pointing_list(input_dir=config["input_dir"], + flag_weight_dir=config["flag_weight_dir"], + pointing_label_list=config["pointing_label_list"], + output_dir=config["output_dir"], + chip_label_list=config["chip_label_list"]) -- GitLab