added clear_dir option and updated get_aux methods

csst_common/data_manager.py

@@ -136,6 +136,7 @@ class CsstMsDataManager:
                  _l0_post: str = "L0_1",
                  log_ppl="csst-l1ppl.log",
                  log_mod="csst-l1mod.log",
+                 clear_dir=False,
                  ):
 
         # version
@@ -200,6 +201,9 @@ class CsstMsDataManager:
 
         # change to working directory
         os.chdir(self.dir_l1)
+        # clear dir_l1
+        if clear_dir:
+            self.clear_dir(self.dir_l1)
 
         # pipeline logger
         self.logger_ppl = get_logger(name="CSST L1 Pipeline Logger", filename=os.path.join(dir_l1, log_ppl))
@@ -236,7 +240,7 @@ class CsstMsDataManager:
         raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!")
 
     @staticmethod
-    def from_dfs(ver_sim="C5.2", datatype="mbi", exp_id=10000100, dir_l1="."):
+    def from_dfs(expid, ver_sim="C5.2", datatype="mbi", dir_l1="."):
         raise NotImplementedError("from_dfs is currently not available!")
 
     @staticmethod
@@ -364,24 +368,21 @@ class CsstMsDataManager:
     def get_bias(self, detector=6):
         """ get bias data """
         if self.datatype == "mbi":
-            fp = glob.glob(self.path_aux.format("CLB", detector))[0]
-            return fits.getdata(fp)
+            return glob.glob(self.path_aux.format("CLB", detector))[0]
         else:
             return self.path_aux.format(detector, "bias")
 
     def get_dark(self, detector=6):
         """ get dark data """
         if self.datatype == "mbi":
-            fp = glob.glob(self.path_aux.format("CLD", detector))[0]
-            return fits.getdata(fp)
+            return glob.glob(self.path_aux.format("CLD", detector))[0]
         else:
             return self.path_aux.format(detector, "dark")
 
     def get_flat(self, detector=6):
         """ get flat data """
         if self.datatype == "mbi":
-            fp = glob.glob(self.path_aux.format("CLF", detector))[0]
-            return fits.getdata(fp)
+            return glob.glob(self.path_aux.format("CLF", detector))[0]
         else:
             return self.path_aux.format(detector, "flat")
 
@@ -430,8 +431,10 @@ class CsstMsDataManager:
         return info
 
     @staticmethod
-    def quickstart(ver_sim="C5.2", datatype="mbi", dir_l1=".", exposure_id=100, dfs_mode=False, dfs_node="kmust"):
-        """Quick dataset generator for tests on dandelion or PMO
+    def quickstart(ver_sim="C5.2", datatype="mbi", dir_l1=".", exposure_id=100,
+                   dfs_mode=False, dfs_node="kmust", clear_dir=False):
+        """
+        Quick dataset generator for tests on dandelion or PML
 
         Parameters
         ----------
@@ -447,6 +450,8 @@ class CsstMsDataManager:
             If True, use DFS.
         dfs_node : str
             The DFS node. Defaults to "kmust", could be "pml".
+        clear_dir : bool
+            If True, clear dir_l1.
 
         Returns
         -------
@@ -487,15 +492,15 @@ class CsstMsDataManager:
     def __repr__(self):
         lines = ""
         lines += "<CsstMsDataManager>\n"
-        lines += f"Data type = {self.datatype}\n"
-        lines += f"Valid detectors = {self.valid_detectors}\n"
-        lines += f"Available detectors = {self.available_detectors}\n"
-        lines += f"Target detectors = {self.target_detectors}\n"
-        lines += f"dir_l0 = {self.dir_l0}\n"
-        lines += f"dir_l1 = {self.dir_l1}\n"
-        lines += f"dfs_mode = {self.dfs_mode}\n"
-        lines += f"dfs_node = {self.dfs_node}\n"
-        lines += f"CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n"
+        lines += f"- Data type = {self.datatype}\n"
+        lines += f"- Valid detectors = {self.valid_detectors}\n"
+        lines += f"- Available detectors = {self.available_detectors}\n"
+        lines += f"- Target detectors = {self.target_detectors}\n"
+        lines += f"- dir_l0 = {self.dir_l0}\n"
+        lines += f"- dir_l1 = {self.dir_l1}\n"
+        lines += f"- dfs_mode = {self.dfs_mode}\n"
+        lines += f"- dfs_node = {self.dfs_node}\n"
+        lines += f"- CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n"
         return lines
 
     def remove_files(self, fmt="*.fits"):
@@ -523,8 +528,8 @@ class CsstMsDataManager:
         for k, v in DFS_CONF[dfs_node].items():
             os.environ[k] = v
 
-    @property
-    def dfs_is_available(self):
+    @staticmethod
+    def dfs_is_available():
         """ Test if DFS works. """
         try:
             tbl = gaia_query_from_dfs(radius=0.001)
@@ -553,10 +558,10 @@ class CsstMsDataManager:
         """ Push SLS spectra to DFS. """
         pass
 
-    def dfs_l0_query(self):
+    def dfs_l0_query(self, obs_id):
         """ Query L0 data from DFS. """
         pass
 
-    def dfs_l1_query(self):
+    def dfs_l1_query(self, obs_id, detector):
         """ Query L1 data from DFS. """
         pass