Skip to content
GitLab
Commit 568d0740 authored by Fang Yuedong's avatar Fang Yuedong
Browse files

add deep_cr module

parent 3b9f0e28
Hide whitespace changes
Inline Side-by-side
measurement_pipeline/L1_pipeline/csst_msc_instrument/image/cosmicray/deepcr.py 0 → 100644
View file @ 568d0740
"""
Identifier: csst-l1/msc/csst_msc_common/csst_msc_common/cosmicray/deepcr.py
Name: deepcr.py
Description: Cosmic ray correction with DeepCR.
Author: Tianmeng Zhang (zhangtm@nao.cas.cn), Xie Zhou (xiezhou@cnlab.net)
Created: 2023-11-16
Modified-History:
2023-11-16, Tianmeng Zhang, created
2023-11-17, Xie Zhou, add lacosmic
2023-11-22, Li Shao, split from common_cr
"""
import os
import numpy as np
import deepCR
from ..config import FlagConfig
__all__ = ["remove_cr_deepcr", ]
MODEL_DIR = os.path.join(os.path.dirname(
os.path.abspath(__file__)), 'cr_model')
DEEPCR_MODEL_PATH = {
'01': '2022-04-26_15_samples_32_layers_epoch30.pth',
'02': '2022-04-26_15_samples_32_layers_epoch30.pth',
'03': '2022-04-26_15_samples_32_layers_epoch30.pth',
'04': '2022-04-26_15_samples_32_layers_epoch30.pth',
'05': '2022-04-26_15_samples_32_layers_epoch30.pth',
'06': 'CSST-CCD06-2023-06-08_hidden32_epoch50.pth',
'07': 'CSST-CCD07-2023-06-08_hidden32_epoch50.pth',
'08': 'CSST-CCD08-2023-06-08_hidden32_epoch50.pth',
'09': 'CSST-CCD09-2023-06-08_hidden32_epoch50.pth',
'10': '2022-04-26_15_samples_32_layers_epoch30.pth',
'11': 'CSST-CCD11-2023-06-07_hidden32_epoch50.pth',
'12': 'CSST-CCD12-2023-06-07_hidden32_epoch50.pth',
'13': 'CSST-CCD13-2023-06-07_hidden32_epoch50.pth',
'14': 'CSST-CCD14-2023-06-07_hidden32_epoch50.pth',
'15': 'CSST-CCD15-2023-06-06_hidden32_epoch50.pth',
'16': 'CSST-CCD16-2023-06-06_hidden32_epoch50.pth',
'17': 'CSST-CCD17-2023-06-06_hidden32_epoch50.pth',
'18': 'CSST-CCD18-2023-06-05_hidden32_epoch50.pth',
'19': 'CSST-CCD19-2023-06-04_hidden32_epoch50.pth',
'20': 'CSST-CCD20-2023-06-05_hidden32_epoch50.pth',
'21': '2022-04-26_15_samples_32_layers_epoch30.pth',
'22': 'CSST-CCD22-2023-06-05_hidden32_epoch50.pth',
'23': 'CSST-CCD23-2023-04-27_hidden32_epoch50.pth',
'24': 'CSST-CCD24-2023-06-05_hidden32_epoch50.pth',
'25': 'CSST-CCD25-2023-06-05_hidden32_epoch50.pth',
'26': '2022-04-26_15_samples_32_layers_epoch30.pth',
'27': '2022-04-26_15_samples_32_layers_epoch30.pth',
'28': '2022-04-26_15_samples_32_layers_epoch30.pth',
'29': '2022-04-26_15_samples_32_layers_epoch30.pth',
'30': '2022-04-26_15_samples_32_layers_epoch30.pth', }
def remove_cr_deepcr(image: np.ndarray,
flag: np.ndarray,
device: str,
chipid: str,
) -> tuple[np.ndarray, np.ndarray, int]:
"""
Remove cosmic rays.
Remove cosmic rays.
Parameters
----------
image : numpy.ndarray
Image to be processed.
flag : numpy.ndarray
Input bitflag map.
device : str
Processing unit: "CPU" or "GPU".
chipid : str
CCD chip id, like "06" or "25", header['CHIPID'].
Returns
-------
image : numpy.ndarray
Processed image.
flag : numpy.ndarray
Added bitflag map.
cr_count : int
Number of flagged cosmic rays.
"""
inpaint = False
inpaint_model = 'ACS-WFC-F606W-2-32'
model_path = os.path.join(MODEL_DIR, DEEPCR_MODEL_PATH[chipid])
model = deepCR.deepCR(
model_path,
inpaint_model,
device=device.upper(),
hidden=32
)
print("Start running deepCR...")
results = model.clean(
image,
threshold=0.5,
inpaint=False,
binary=True,
segment=True,
patch=256,
parallel=False,
n_jobs=1
)
# results = model.clean(
# image,
# threshold=0.5,
# inpaint=False,
# binary=True,
# segment=True,
# patch=1024,
# parallel=False,
# n_jobs=-1
# )
if inpaint:
_flag, image = results
else:
_flag = results
flagconf = FlagConfig()
_flag = _flag.astype(flagconf.dtype)
value = flagconf.get_value('cosmic_ray')
flag = np.bitwise_or(flag, _flag * value)
cr_count = int(np.sum(_flag))
return image, flag, cr_count
measurement_pipeline/L1_pipeline/csst_msc_instrument/image/cosmicray/lacosmic.py 0 → 100644
View file @ 568d0740
"""
Identifier: csst-l1/msc/csst_msc_common/csst_msc_common/cosmicray/lacosmic.py
Name: lacosmic.py
Description: Cosmic ray correction with L.A.Cosmic.
Author: Xie Zhou (xiezhou@cnlab.net), Tianmeng Zhang (zhangtm@nao.cas.cn)
Created: 2023-11-16
Modified-History:
2023-11-16, Tianmeng Zhang, created
2023-11-17, Xie Zhou, add lacosmic
2023-11-22, Li Shao, split from common_cr, improve code format
"""
import numpy as np
from ccdproc import cosmicray_lacosmic
from ..config import FlagConfig
__all__ = ["remove_cr_la", ]
def remove_cr_la(image: np.ndarray,
flag: np.ndarray,
gain: float = 1.1,
readnoise: float = 5.0,
sigclip: float = 3.0,
sigfrac: float = 0.5,
objlim: float = 5.0,
satlevel: float = 65535.0,
pssl: float = 0.0,
niter: int = 4,
sepmed: bool = True,
cleantype: str = 'meanmask',
fsmode: str = 'median',
psfmodel: str = 'gauss',
psffwhm: float = 2.5,
psfsize: int = 7,
psfk: np.ndarray | None = None,
psfbeta: float = 4.765,
verbose: bool = False,
gain_apply: bool = True,
) -> tuple[np.ndarray, np.ndarray, int]:
"""
Remove cosmic rays with L.A.Cosmic algorithm.
Remove cosmic rays with L.A.Cosmic algorithm.
Parameters
----------
image : numpy.ndarray
Image to be processed.
flag : numpy.ndarray
Input bitflag map.
gain : float or astropy.units.Quantity, optional
Gain of the image (electrons / ADU). Default: 1.0.
readnoise : float, optional
Read noise of the image (electrons). Used to generate the noise model of the image. Default: 6.5.
sigclip : float, optional
Laplacian-to-noise limit for cosmic ray detection.
Lower values will flag more pixels as cosmic rays. Default: 4.5.
sigfrac : float, optional
Fractional detection limit for neighboring pixels. For cosmic ray neighbor pixels,
a Laplacian-to-noise detection limit of sigfrac * sigclip will be used. Default: 0.3.
objlim : float, optional
Minimum contrast between Laplacian image and the fine structure image. Increase this value if
cores of bright stars are flagged as cosmic rays. Default: 5.0.
satlevel : float, optional
Saturation level of the image (electrons). This value is used to detect saturated stars and pixels
at or above this level are added to the mask. Default: 65535.0.
pssl : float, optional
Previously subtracted sky level in ADU. Default: 0.0.
niter : int, optional
Number of iterations of the LA Cosmic algorithm to perform. Default: 4.
sepmed : bool, optional
Use the separable median filter instead of the full median filter. The separable median is not
identical to the full median filter, but they are approximately the same, the separable median
filter is significantly faster, and still detects cosmic rays well. Note, this is a performance
feature, and not part of the original L.A. Cosmic. Default: True.
cleantype : str, optional
Set which clean algorithm is used:
- "median": An unmasked 5x5 median filter.
- "medmask": A masked 5x5 median filter.
- "meanmask": A masked 5x5 mean filter.
- "idw": A masked 5x5 inverse distance weighted interpolation.
Default: "meanmask".
fsmode : str, optional
Method to build the fine structure image:
- "median": Use the median filter in the standard LA Cosmic algorithm.
- "convolve": Convolve the image with the psf kernel to calculate the fine structure image.
Default: "median".
psfmodel : str, optional
Model to use to generate the psf kernel if fsmode == 'convolve' and psfk is None. The current
choices are Gaussian and Moffat profiles:
- "gauss" and "moffat" produce circular PSF kernels.
- The "gaussx"`` and "gaussy" produce Gaussian kernels in the x and y directions respectively.
Default: "gauss".
psffwhm : float, optional
Full Width Half Maximum of the PSF to use to generate the kernel. Default: 2.5.
psfsize : int, optional
Size of the kernel to calculate. Returned kernel will have size psfsize x psfsize. psfsize should be odd.
Default: 7.
psfk : numpy.ndarray[float] or None, optional
PSF kernel array to use for the fine structure image if fsmode == "convolve".
If None and fsmode == "convolve", calculate the psf kernel using "psfmodel". Default: None.
psfbeta : float, optional
Moffat beta parameter. Only used if fsmode=="convolve" and psfmodel=="moffat". Default: 4.765.
verbose : bool, optional
Print to the screen or not. Default: False.
gain_apply : bool, optional
If True, return gain-corrected data, with correct units, otherwise do not gain-correct the data.
Default is "True" to preserve backwards compatibility.
Returns
-------
image : numpy.ndarray
Processed image.
flag : numpy.ndarray
Added bitflag map.
cr_count : int
Number of flagged cosmic rays.
"""
image, _flag = cosmicray_lacosmic(ccd=image,
sigclip=sigclip,
sigfrac=sigfrac,
objlim=objlim,
gain=gain,
readnoise=readnoise,
satlevel=satlevel,
pssl=pssl,
niter=niter,
sepmed=sepmed,
cleantype=cleantype,
fsmode=fsmode,
psfmodel=psfmodel,
psffwhm=psffwhm,
psfsize=psfsize,
psfk=psfk,
psfbeta=psfbeta,
verbose=verbose,
gain_apply=gain_apply)
flagconf = FlagConfig()
_flag = _flag.astype(flagconf.dtype)
value = flagconf.get_value('cosmic_ray')
flag = np.bitwise_or(flag, _flag * value)
cr_count = np.sum(_flag)
return image, flag, cr_count
measurement_pipeline/run_csst_msc_instrument.py
View file @ 568d0740
......@@ -120,13 +120,15 @@ def run_pointing_list(input_dir,
if __name__ == "__main__":
input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W1/"
pointing_label_list = ["MSC_0000000", "MSC_0000001",
"MSC_0000002", "MSC_0000003", "MSC_0000004", "MSC_0000005"]
# input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W1/"
# pointing_label_list = ["MSC_0000000"]
# chip_label_list = ["08"]
# output_dir = "/public/home/fangyuedong/project/test_deepcr"
input_dir = "/public/share/yangxuliu/CSSOSDataProductsSims/outputs_50sqDeg/50sqDeg_Photo_W2/"
pointing_label_list = ["MSC_0000000", "MSC_0000001",
"MSC_0000002", "MSC_0000003", "MSC_0000004", "MSC_0000005", "MSC_0000006", "MSC_0000007", "MSC_0000008", "MSC_0000009"]
chip_label_list = None
output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs"
output_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs/50sqDeg_Photo_W2/"
run_pointing_list(input_dir=input_dir,
pointing_label_list=pointing_label_list,
......
measurement_pipeline/run_csst_msc_mbi_photometry.py
View file @ 568d0740
......@@ -129,17 +129,16 @@ def run_pointing_list(input_dir,
if __name__ == "__main__":
# input_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs"
input_dir = "/public/home/fangyuedong/project/injected_50sqDeg_L1_outputs"
pointing_label_list = ["MSC_0000000", "MSC_0000001",
"MSC_0000002", "MSC_0000003", "MSC_0000004", "MSC_0000005"]
chip_label_list = None
# output_dir = "/public/home/fangyuedong/project/test_photometry"
# input_dir = "/public/home/fangyuedong/project/injected_50sqDeg_L1_outputs"
# pointing_label_list = ["MSC_0000000", "MSC_0000001",
# "MSC_0000002", "MSC_0000003", "MSC_0000004", "MSC_0000005"]
# chip_label_list = None
output_dir = "/public/home/fangyuedong/project/processed_injected_50sqDeg_L1_outputs"
flag_weight_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs"
# pointing_label_list = ["MSC_0000000"]
# chip_label_list = ["08"]
input_dir = "/public/home/fangyuedong/project/50sqDeg_L1_outputs"
pointing_label_list = ["MSC_0000000"]
chip_label_list = ["08"]
# output_dir = "/public/home/fangyuedong/project/test_photometry"
run_pointing_list(input_dir=input_dir,
flag_weight_dir=flag_weight_dir,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment