Skip to content
GitLab
Commit c283cd15 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

compatible with new L0 data model

parent 2ce2cdbd
Hide whitespace changes
Inline Side-by-side
csst_common/data/csst_params.yml
View file @ c283cd15
......@@ -77,6 +77,7 @@ sls: # slit-less spectra
sim: # simulation
versions:
- C6.2
- C6.1
- C5.2
......
csst_common/data_manager.py
View file @ c283cd15
......@@ -13,7 +13,6 @@ Modified-History:
2022-11-06, Bo Zhang, deleted CsstMbiDataManager
2022-11-20, Bo Zhang, added DFS APIs
"""
import datetime
import glob
import os
import re
......@@ -93,15 +92,15 @@ class CsstMsDataManager:
The exposure start time in ``yyyymmddhhmmss`` format.
exp_stop : int
The exposure start time in ``yyyymmddhhmmss`` format.
_telescope : str
telescope : str
The telescope name. Defaults to ``CSST`` for C5.2 simulation.
_instrument : str
instrument : str
The instrument name. Defaults to ``MSC`` for C5.2 simulation.
_survey : str
project : str
The survey name. Defaults to ``MS`` for C5.2 simulation.
obs_type : str
The image type signature for science images. Defualts to ``SCI`` for C5.2 simulation.
l0_post : str
level : str
The postfix. Defaults to ``L0_1`` for C5.2 simulation.
log_ppl : str
The pipeline log file name.
......@@ -124,27 +123,26 @@ class CsstMsDataManager:
Examples
--------
>>> dm_mbi = CsstMsDataManager(...)
>>> dm = CsstMsDataManager.from_dir(
>>> ver_sim="C6.2", datatype="mbi", dir_l0="/Users/cham/L1Pipeline/L0_MBI_C6.2", dir_l1=".")
>>> # access L0 directory
>>> dm_mbi.dir_l0
>>> dm.dir_l0
>>> # access L1 directory
>>> dm_mbi.dir_l1
>>> dm.dir_l1
>>> # access path_aux
>>> dm_mbi.path_aux
>>> # access ver_sim
>>> dm_mbi.ver_sim
>>> dm.path_aux
>>> # access target detectors
>>> dm_mbi.target_detectors
>>> dm.target_detectors
>>> # access available detectors
>>> dm_mbi.available_detectors
>>> # define an L1 file (detector-specified)
>>> dm_mbi.l1_detector(detector=6)
>>> dm.available_detectors
>>> # generate an L1 file path (detector-specified)
>>> dm.l1_detector(detector=6, post="IMG.fits")
>>> # define an L1 file (non-detector-specified)
>>> dm_mbi.l1_file("flipped_image.fits")
>>> dm.l1_file("flipped_image.fits")
"""
# TODO: update docstring
def __init__(self,
ver_sim: str = "C5.2",
ver_sim: str = "C6.2",
datatype: str = "mbi",
available_detectors: Union[None, list] = None,
target_detectors: Union[None, list, int] = None,
......@@ -154,14 +152,16 @@ class CsstMsDataManager:
use_dfs: bool = False,
dfs_node: str = "kmust",
dfs_root: str = "/share/dfs",
obs_id: str = "100000100",
telescope: str = "CSST",
instrument: str = "MSC",
project: str = "MS",
obs_type: str = "SCI",
exp_start: int = "20270810081950",
exp_stop: int = "20270810082220",
_telescope: str = "CSST",
_instrument: str = "MSC",
_survey: str = "MS",
obs_type: str = "SCI",
l0_post: str = "L0_1",
obs_id: str = "100000100",
level: str = "0",
version: str = "01",
ext: str = "fits",
log_ppl: str = "",
log_mod: str = "",
clear_dir: bool = False,
......@@ -214,11 +214,13 @@ class CsstMsDataManager:
self.exp_stop = exp_stop
# file name components
self._telescope = _telescope
self._instrument = _instrument
self._survey = _survey
self.telescope = telescope
self.instrument = instrument
self.project = project
self.obs_type = obs_type
self.l0_post = l0_post
self.level = level
self.version = version
self.ext = ext
# DFS configuration
self.use_dfs = use_dfs
......@@ -353,11 +355,12 @@ class CsstMsDataManager:
@staticmethod
def from_dir(
ver_sim="C5.2",
ver_sim="C6.2",
datatype="mbi",
dir_l0=".",
dir_l1=".",
path_aux="",
pattern="CSST_MSC_*_SCI_*.fits",
use_dfs=False,
dfs_node="kmust",
log_ppl="csst-l1ppl.log",
......@@ -372,23 +375,33 @@ class CsstMsDataManager:
assert ver_sim in CP["sim"]["versions"]
# glob files
fps_img = CsstMsDataManager.glob_image(dir_l0, ver_sim=ver_sim)
fps_img = CsstMsDataManager.glob_image(dir_l0=dir_l0, pattern=pattern)
if len(fps_img) == 0:
raise FileNotFoundError(f"No file found in dir_l0: {dir_l0}")
raise FileNotFoundError(f"No file found with pattern {pattern} in {dir_l0}")
# parse filename
pattern = re.compile(
r"(?P<telescope>[A-Z]+)_"
r"(?P<instrument>[A-Z]+)_"
r"(?P<project>[A-Z]+)_"
r"(?P<obs_type>[A-Z]+)_"
r"(?P<exp_start>[0-9]{14})_"
r"(?P<exp_stop>[0-9]{14})_"
r"(?P<obs_id>[0-9]{11})_"
r"(?P<detector>[0-9]{2})_"
r"L(?P<level>[0-9]{1})_"
r"V(?P<version>[0-9]{2})"
r".(?P<ext>[a-z]{4})"
)
mo = re.fullmatch(pattern, fps_img[0])
assert mo is not None
mogd = mo.groupdict()
mogd.pop("detector")
# available detectors
available_detectors = [int(re.split(r"[_.]", fp)[7]) for fp in fps_img]
available_detectors = [int(re.fullmatch(pattern, fp)["detector"]) for fp in fps_img]
available_detectors.sort()
# parse info
(_telescope, _instrument, _survey, obs_type,
exp_start, exp_stop, obs_id,
_detector, *l0_post, _ext) = re.split(r"[_.]", fps_img[0])
# exp_start = int(exp_start)
# exp_stop = int(exp_stop)
# obs_id = int(obs_id)
return CsstMsDataManager(
ver_sim=ver_sim,
datatype=datatype,
......@@ -399,81 +412,63 @@ class CsstMsDataManager:
path_aux=path_aux, # bias dark flat
use_dfs=use_dfs,
dfs_node=dfs_node,
obs_id=obs_id,
exp_start=exp_start,
exp_stop=exp_stop,
_telescope=_telescope,
_instrument=_instrument,
_survey=_survey,
obs_type=obs_type,
l0_post="_".join(l0_post),
log_ppl=log_ppl,
log_mod=log_mod,
n_jobs=n_jobs,
backend=backend,
device=device,
**mogd,
**kwargs
)
@staticmethod
def glob_image(dir_l0, ver_sim="C5.2"):
def glob_image(dir_l0, pattern="CSST_MSC_*_SCI_*.fits"):
""" glob files in L0 data directory """
assert ver_sim in CP["sim"]["versions"]
pattern = os.path.join(dir_l0, "CSST_MSC_MS_SCI_*.fits")
fps = glob.glob(pattern)
fps = glob.glob(os.path.join(dir_l0, pattern))
fps = [os.path.basename(fp) for fp in fps]
fps.sort()
print("{} files found with pattern: {}".format(len(fps), pattern))
return fps
@staticmethod
def glob_cat(dir_l0, ver_sim="C5"):
""" glob input catalogs in L0 data directory """
assert ver_sim in CP["sim"]["versions"]
pattern = os.path.join(dir_l0, "MSC_*.cat")
fps = glob.glob(pattern)
fps = [os.path.basename(fp) for fp in fps]
fps.sort()
print("@DM.glob_dir: {} files found with pattern: {}".format(len(fps), pattern))
return fps
def l0_id(self, detector=6):
""" Level0 ID, consistent with DFS. """
return f"{self.obs_id}{detector:02d}"
def l0_cat(self, detector=6):
""" the L0 cat file path"""
assert self.ver_sim in CP["sim"]["versions"]
fn = "{}_{}_chip_{:02d}_filt_{}.cat".format(
self._instrument, self.obs_id, detector, self.detector2filter[detector])
return os.path.join(self.dir_l0, fn)
assert detector in self.available_detectors
fn = f"{self.instrument}_{self.obs_id}_chip_{detector:02d}_filt_{self.detector2filter[detector]}.cat"
fp = os.path.join(self.dir_l0, fn)
assert os.path.exists(fp)
return fp
def l0_log(self, detector=6):
""" L0 log file path """
assert self.ver_sim in CP["sim"]["versions"]
fn = "{}_{}_chip_{:02d}_filt_{}.log".format(
self._instrument, self.obs_id, detector, self.detector2filter[detector])
return os.path.join(self.dir_l0, fn)
assert detector in self.available_detectors
fn = f"{self.instrument}_{self.obs_id}_chip_{detector:02d}_filt_{self.detector2filter[detector]}.log"
fp = os.path.join(self.dir_l0, fn)
assert os.path.exists(fp)
return fp
def l0_detector(self, detector=6):
""" L0 detector-specific image file path """
assert self.ver_sim in CP["sim"]["versions"]
fn = "{}_{}_{}_SCI_{}_{}_{}_{:02d}_L0_1.fits".format(
self._telescope, self._instrument, self._survey,
self.exp_start, self.exp_stop, self.obs_id, detector)
assert detector in self.available_detectors
fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
f"{self.obs_id}_{detector:02d}_L{self.level}_V{self.version}.{self.ext}"
fp = os.path.join(self.dir_l0, fn)
assert os.path.exists(fp)
return os.path.join(self.dir_l0, fn)
def l0_crs(self, detector=6):
""" L0 cosmic ray file path """
assert self.ver_sim in CP["sim"]["versions"]
fn = "{}_{}_{}_CRS_{}_{}_{}_{:02d}_L0_1.fits".format(
self._telescope, self._instrument, self._survey,
self.exp_start, self.exp_stop, self.obs_id, detector)
assert detector in self.available_detectors
fn = f"{self.telescope}_{self.instrument}_{self.project}_CRS_{self.exp_start}_{self.exp_stop}_" \
f"{self.obs_id}_{detector:02d}_L{self.level}_V{self.version}.{self.ext}"
fp = os.path.join(self.dir_l0, fn)
assert os.path.exists(fp)
return os.path.join(self.dir_l0, fn)
def l1_detector(self, detector=6, post="img.fits"):
def l1_detector(self, detector=6, post="IMG.fits"):
""" generate L1 file path
Parameters
......@@ -489,10 +484,9 @@ class CsstMsDataManager:
L1 file path
"""
assert self.ver_sim in CP["sim"]["versions"]
fn = "{}_{}_{}_SCI_{}_{}_{}_{:02d}_{}".format(
self._telescope, self._instrument, self._survey,
self.exp_start, self.exp_stop, self.obs_id, detector, post)
assert detector in self.available_detectors
fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
f"{self.obs_id}_{detector:02d}_L1_V{self.version}_{post}"
return os.path.join(self.dir_l1, fn)
def get_bias(self, detector=6):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment