compatible with new L0 data model

csst_common/data/csst_params.yml

@@ -77,6 +77,7 @@ sls: # slit-less spectra
 
 sim: # simulation
   versions:
+    - C6.2
     - C6.1
     - C5.2
 

csst_common/data_manager.py

@@ -13,7 +13,6 @@ Modified-History:
     2022-11-06, Bo Zhang, deleted CsstMbiDataManager
     2022-11-20, Bo Zhang, added DFS APIs
 """
-import datetime
 import glob
 import os
 import re
@@ -93,15 +92,15 @@ class CsstMsDataManager:
         The exposure start time in ``yyyymmddhhmmss`` format.
     exp_stop : int
         The exposure start time in ``yyyymmddhhmmss`` format.
-    _telescope : str
+    telescope : str
         The telescope name. Defaults to ``CSST`` for C5.2 simulation.
-    _instrument : str
+    instrument : str
         The instrument name. Defaults to ``MSC`` for C5.2 simulation.
-    _survey : str
+    project : str
         The survey name. Defaults to ``MS`` for C5.2 simulation.
     obs_type : str
         The image type signature for science images. Defualts to ``SCI`` for C5.2 simulation.
-    l0_post : str
+    level : str
         The postfix. Defaults to ``L0_1`` for C5.2 simulation.
     log_ppl : str
         The pipeline log file name.
@@ -124,27 +123,26 @@ class CsstMsDataManager:
 
     Examples
     --------
-    >>> dm_mbi = CsstMsDataManager(...)
+    >>> dm = CsstMsDataManager.from_dir(
+    >>>     ver_sim="C6.2", datatype="mbi", dir_l0="/Users/cham/L1Pipeline/L0_MBI_C6.2", dir_l1=".")
     >>> # access L0 directory
-    >>> dm_mbi.dir_l0
+    >>> dm.dir_l0
     >>> # access L1 directory
-    >>> dm_mbi.dir_l1
+    >>> dm.dir_l1
     >>> # access path_aux
-    >>> dm_mbi.path_aux
-    >>> # access ver_sim
-    >>> dm_mbi.ver_sim
+    >>> dm.path_aux
     >>> # access target detectors
-    >>> dm_mbi.target_detectors
+    >>> dm.target_detectors
     >>> # access available detectors
-    >>> dm_mbi.available_detectors
-    >>> # define an L1 file (detector-specified)
-    >>> dm_mbi.l1_detector(detector=6)
+    >>> dm.available_detectors
+    >>> # generate an L1 file path (detector-specified)
+    >>> dm.l1_detector(detector=6, post="IMG.fits")
     >>> # define an L1 file (non-detector-specified)
-    >>> dm_mbi.l1_file("flipped_image.fits")
+    >>> dm.l1_file("flipped_image.fits")
     """
-
+    # TODO: update docstring
     def __init__(self,
-                 ver_sim: str = "C5.2",
+                 ver_sim: str = "C6.2",
                  datatype: str = "mbi",
                  available_detectors: Union[None, list] = None,
                  target_detectors: Union[None, list, int] = None,
@@ -154,14 +152,16 @@ class CsstMsDataManager:
                  use_dfs: bool = False,
                  dfs_node: str = "kmust",
                  dfs_root: str = "/share/dfs",
-                 obs_id: str = "100000100",
+                 telescope: str = "CSST",
+                 instrument: str = "MSC",
+                 project: str = "MS",
+                 obs_type: str = "SCI",
                  exp_start: int = "20270810081950",
                  exp_stop: int = "20270810082220",
-                 _telescope: str = "CSST",
-                 _instrument: str = "MSC",
-                 _survey: str = "MS",
-                 obs_type: str = "SCI",
-                 l0_post: str = "L0_1",
+                 obs_id: str = "100000100",
+                 level: str = "0",
+                 version: str = "01",
+                 ext: str = "fits",
                  log_ppl: str = "",
                  log_mod: str = "",
                  clear_dir: bool = False,
@@ -214,11 +214,13 @@ class CsstMsDataManager:
         self.exp_stop = exp_stop
 
         # file name components
-        self._telescope = _telescope
-        self._instrument = _instrument
-        self._survey = _survey
+        self.telescope = telescope
+        self.instrument = instrument
+        self.project = project
         self.obs_type = obs_type
-        self.l0_post = l0_post
+        self.level = level
+        self.version = version
+        self.ext = ext
 
         # DFS configuration
         self.use_dfs = use_dfs
@@ -353,11 +355,12 @@ class CsstMsDataManager:
 
     @staticmethod
     def from_dir(
-            ver_sim="C5.2",
+            ver_sim="C6.2",
             datatype="mbi",
             dir_l0=".",
             dir_l1=".",
             path_aux="",
+            pattern="CSST_MSC_*_SCI_*.fits",
             use_dfs=False,
             dfs_node="kmust",
             log_ppl="csst-l1ppl.log",
@@ -372,23 +375,33 @@ class CsstMsDataManager:
         assert ver_sim in CP["sim"]["versions"]
 
         # glob files
-        fps_img = CsstMsDataManager.glob_image(dir_l0, ver_sim=ver_sim)
+        fps_img = CsstMsDataManager.glob_image(dir_l0=dir_l0, pattern=pattern)
         if len(fps_img) == 0:
-            raise FileNotFoundError(f"No file found in dir_l0: {dir_l0}")
+            raise FileNotFoundError(f"No file found with pattern {pattern} in {dir_l0}")
+
+        # parse filename
+        pattern = re.compile(
+            r"(?P<telescope>[A-Z]+)_"
+            r"(?P<instrument>[A-Z]+)_"
+            r"(?P<project>[A-Z]+)_"
+            r"(?P<obs_type>[A-Z]+)_"
+            r"(?P<exp_start>[0-9]{14})_"
+            r"(?P<exp_stop>[0-9]{14})_"
+            r"(?P<obs_id>[0-9]{11})_"
+            r"(?P<detector>[0-9]{2})_"
+            r"L(?P<level>[0-9]{1})_"
+            r"V(?P<version>[0-9]{2})"
+            r".(?P<ext>[a-z]{4})"
+        )
+        mo = re.fullmatch(pattern, fps_img[0])
+        assert mo is not None
+        mogd = mo.groupdict()
+        mogd.pop("detector")
 
         # available detectors
-        available_detectors = [int(re.split(r"[_.]", fp)[7]) for fp in fps_img]
+        available_detectors = [int(re.fullmatch(pattern, fp)["detector"]) for fp in fps_img]
         available_detectors.sort()
 
-        # parse info
-        (_telescope, _instrument, _survey, obs_type,
-         exp_start, exp_stop, obs_id,
-         _detector, *l0_post, _ext) = re.split(r"[_.]", fps_img[0])
-
-        # exp_start = int(exp_start)
-        # exp_stop = int(exp_stop)
-        # obs_id = int(obs_id)
-
         return CsstMsDataManager(
             ver_sim=ver_sim,
             datatype=datatype,
@@ -399,81 +412,63 @@ class CsstMsDataManager:
             path_aux=path_aux,  # bias dark flat
             use_dfs=use_dfs,
             dfs_node=dfs_node,
-            obs_id=obs_id,
-            exp_start=exp_start,
-            exp_stop=exp_stop,
-            _telescope=_telescope,
-            _instrument=_instrument,
-            _survey=_survey,
-            obs_type=obs_type,
-            l0_post="_".join(l0_post),
             log_ppl=log_ppl,
             log_mod=log_mod,
             n_jobs=n_jobs,
             backend=backend,
             device=device,
+            **mogd,
             **kwargs
         )
 
     @staticmethod
-    def glob_image(dir_l0, ver_sim="C5.2"):
+    def glob_image(dir_l0, pattern="CSST_MSC_*_SCI_*.fits"):
         """ glob files in L0 data directory """
-        assert ver_sim in CP["sim"]["versions"]
-        pattern = os.path.join(dir_l0, "CSST_MSC_MS_SCI_*.fits")
-        fps = glob.glob(pattern)
+        fps = glob.glob(os.path.join(dir_l0, pattern))
         fps = [os.path.basename(fp) for fp in fps]
         fps.sort()
-
         print("{} files found with pattern: {}".format(len(fps), pattern))
         return fps
 
-    @staticmethod
-    def glob_cat(dir_l0, ver_sim="C5"):
-        """ glob input catalogs in L0 data directory """
-        assert ver_sim in CP["sim"]["versions"]
-        pattern = os.path.join(dir_l0, "MSC_*.cat")
-        fps = glob.glob(pattern)
-        fps = [os.path.basename(fp) for fp in fps]
-        fps.sort()
-
-        print("@DM.glob_dir: {} files found with pattern: {}".format(len(fps), pattern))
-        return fps
-
     def l0_id(self, detector=6):
         """ Level0 ID, consistent with DFS. """
         return f"{self.obs_id}{detector:02d}"
 
     def l0_cat(self, detector=6):
         """ the L0 cat file path"""
-        assert self.ver_sim in CP["sim"]["versions"]
-        fn = "{}_{}_chip_{:02d}_filt_{}.cat".format(
-            self._instrument, self.obs_id, detector, self.detector2filter[detector])
-        return os.path.join(self.dir_l0, fn)
+        assert detector in self.available_detectors
+        fn = f"{self.instrument}_{self.obs_id}_chip_{detector:02d}_filt_{self.detector2filter[detector]}.cat"
+        fp = os.path.join(self.dir_l0, fn)
+        assert os.path.exists(fp)
+        return fp
 
     def l0_log(self, detector=6):
         """ L0 log file path """
-        assert self.ver_sim in CP["sim"]["versions"]
-        fn = "{}_{}_chip_{:02d}_filt_{}.log".format(
-            self._instrument, self.obs_id, detector, self.detector2filter[detector])
-        return os.path.join(self.dir_l0, fn)
+        assert detector in self.available_detectors
+        fn = f"{self.instrument}_{self.obs_id}_chip_{detector:02d}_filt_{self.detector2filter[detector]}.log"
+        fp = os.path.join(self.dir_l0, fn)
+        assert os.path.exists(fp)
+        return fp
 
     def l0_detector(self, detector=6):
         """ L0 detector-specific image file path """
-        assert self.ver_sim in CP["sim"]["versions"]
-        fn = "{}_{}_{}_SCI_{}_{}_{}_{:02d}_L0_1.fits".format(
-            self._telescope, self._instrument, self._survey,
-            self.exp_start, self.exp_stop, self.obs_id, detector)
+        assert detector in self.available_detectors
+        fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
+             f"{self.obs_id}_{detector:02d}_L{self.level}_V{self.version}.{self.ext}"
+        fp = os.path.join(self.dir_l0, fn)
+        assert os.path.exists(fp)
         return os.path.join(self.dir_l0, fn)
 
     def l0_crs(self, detector=6):
         """ L0 cosmic ray file path """
-        assert self.ver_sim in CP["sim"]["versions"]
-        fn = "{}_{}_{}_CRS_{}_{}_{}_{:02d}_L0_1.fits".format(
-            self._telescope, self._instrument, self._survey,
-            self.exp_start, self.exp_stop, self.obs_id, detector)
+        assert detector in self.available_detectors
+        fn = f"{self.telescope}_{self.instrument}_{self.project}_CRS_{self.exp_start}_{self.exp_stop}_" \
+             f"{self.obs_id}_{detector:02d}_L{self.level}_V{self.version}.{self.ext}"
+        fp = os.path.join(self.dir_l0, fn)
+        assert os.path.exists(fp)
         return os.path.join(self.dir_l0, fn)
 
-    def l1_detector(self, detector=6, post="img.fits"):
+    def l1_detector(self, detector=6, post="IMG.fits"):
         """ generate L1 file path
 
         Parameters
@@ -489,10 +484,9 @@ class CsstMsDataManager:
         L1 file path
 
         """
-        assert self.ver_sim in CP["sim"]["versions"]
-        fn = "{}_{}_{}_SCI_{}_{}_{}_{:02d}_{}".format(
-            self._telescope, self._instrument, self._survey,
-            self.exp_start, self.exp_stop, self.obs_id, detector, post)
+        assert detector in self.available_detectors
+        fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
+             f"{self.obs_id}_{detector:02d}_L1_V{self.version}_{post}"
         return os.path.join(self.dir_l1, fn)
 
     def get_bias(self, detector=6):