diff --git a/csst_common/data_manager.py b/csst_common/data_manager.py index e3bcfc363729effd1c3a36b6fc74f4c5082d8817..9ffc98ee648f5c999a5e9a821371b1e7837cd434 100644 --- a/csst_common/data_manager.py +++ b/csst_common/data_manager.py @@ -136,6 +136,7 @@ class CsstMsDataManager: _l0_post: str = "L0_1", log_ppl="csst-l1ppl.log", log_mod="csst-l1mod.log", + clear_dir=False, ): # version @@ -200,6 +201,9 @@ class CsstMsDataManager: # change to working directory os.chdir(self.dir_l1) + # clear dir_l1 + if clear_dir: + self.clear_dir(self.dir_l1) # pipeline logger self.logger_ppl = get_logger(name="CSST L1 Pipeline Logger", filename=os.path.join(dir_l1, log_ppl)) @@ -236,7 +240,7 @@ class CsstMsDataManager: raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!") @staticmethod - def from_dfs(ver_sim="C5.2", datatype="mbi", exp_id=10000100, dir_l1="."): + def from_dfs(expid, ver_sim="C5.2", datatype="mbi", dir_l1="."): raise NotImplementedError("from_dfs is currently not available!") @staticmethod @@ -364,24 +368,21 @@ class CsstMsDataManager: def get_bias(self, detector=6): """ get bias data """ if self.datatype == "mbi": - fp = glob.glob(self.path_aux.format("CLB", detector))[0] - return fits.getdata(fp) + return glob.glob(self.path_aux.format("CLB", detector))[0] else: return self.path_aux.format(detector, "bias") def get_dark(self, detector=6): """ get dark data """ if self.datatype == "mbi": - fp = glob.glob(self.path_aux.format("CLD", detector))[0] - return fits.getdata(fp) + return glob.glob(self.path_aux.format("CLD", detector))[0] else: return self.path_aux.format(detector, "dark") def get_flat(self, detector=6): """ get flat data """ if self.datatype == "mbi": - fp = glob.glob(self.path_aux.format("CLF", detector))[0] - return fits.getdata(fp) + return glob.glob(self.path_aux.format("CLF", detector))[0] else: return self.path_aux.format(detector, "flat") @@ -430,8 +431,10 @@ class CsstMsDataManager: return info @staticmethod - def quickstart(ver_sim="C5.2", datatype="mbi", dir_l1=".", exposure_id=100, dfs_mode=False, dfs_node="kmust"): - """Quick dataset generator for tests on dandelion or PMO + def quickstart(ver_sim="C5.2", datatype="mbi", dir_l1=".", exposure_id=100, + dfs_mode=False, dfs_node="kmust", clear_dir=False): + """ + Quick dataset generator for tests on dandelion or PML Parameters ---------- @@ -447,6 +450,8 @@ class CsstMsDataManager: If True, use DFS. dfs_node : str The DFS node. Defaults to "kmust", could be "pml". + clear_dir : bool + If True, clear dir_l1. Returns ------- @@ -487,15 +492,15 @@ class CsstMsDataManager: def __repr__(self): lines = "" lines += "\n" - lines += f"Data type = {self.datatype}\n" - lines += f"Valid detectors = {self.valid_detectors}\n" - lines += f"Available detectors = {self.available_detectors}\n" - lines += f"Target detectors = {self.target_detectors}\n" - lines += f"dir_l0 = {self.dir_l0}\n" - lines += f"dir_l1 = {self.dir_l1}\n" - lines += f"dfs_mode = {self.dfs_mode}\n" - lines += f"dfs_node = {self.dfs_node}\n" - lines += f"CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n" + lines += f"- Data type = {self.datatype}\n" + lines += f"- Valid detectors = {self.valid_detectors}\n" + lines += f"- Available detectors = {self.available_detectors}\n" + lines += f"- Target detectors = {self.target_detectors}\n" + lines += f"- dir_l0 = {self.dir_l0}\n" + lines += f"- dir_l1 = {self.dir_l1}\n" + lines += f"- dfs_mode = {self.dfs_mode}\n" + lines += f"- dfs_node = {self.dfs_node}\n" + lines += f"- CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n" return lines def remove_files(self, fmt="*.fits"): @@ -523,8 +528,8 @@ class CsstMsDataManager: for k, v in DFS_CONF[dfs_node].items(): os.environ[k] = v - @property - def dfs_is_available(self): + @staticmethod + def dfs_is_available(): """ Test if DFS works. """ try: tbl = gaia_query_from_dfs(radius=0.001) @@ -553,10 +558,10 @@ class CsstMsDataManager: """ Push SLS spectra to DFS. """ pass - def dfs_l0_query(self): + def dfs_l0_query(self, obs_id): """ Query L0 data from DFS. """ pass - def dfs_l1_query(self): + def dfs_l1_query(self, obs_id, detector): """ Query L1 data from DFS. """ pass