From a6b8bf1a2a6b58d863c55b2f9402aea123dd52a9 Mon Sep 17 00:00:00 2001 From: BO ZHANG Date: Mon, 21 Nov 2022 11:35:21 +0800 Subject: [PATCH] add verbose option --- csst_common/data_manager.py | 52 +++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/csst_common/data_manager.py b/csst_common/data_manager.py index 3e4e7d4..8a31df0 100644 --- a/csst_common/data_manager.py +++ b/csst_common/data_manager.py @@ -9,7 +9,7 @@ Modified-History: 2022-09-13, Bo Zhang, added CsstMbiDataManager 2022-09-29, Bo Zhang, favor CsstMsDataManager instead of CsstMbiDataManager 2022-10-26, Bo Zhang, reconstruct CsstMsDataManager, deprecate CsstMbiDataManager - 2022-10-28, Bo Zhang, added CsstMsDataManager.query_rc(), dm.dfs_mode, dm.node + 2022-10-28, Bo Zhang, added CsstMsDataManager.query_rc(), dm.use_dfs, dm.node 2022-11-06, Bo Zhang, deleted CsstMbiDataManager 2022-11-20, Bo Zhang, added DFS APIs """ @@ -75,8 +75,8 @@ class CsstMsDataManager: The L1 directory. path_aux : str The aux data directory (bias, flat, dark). - dfs_mode : bool - DFS mode. If True, ``CsstMsDataManager`` will use DFS. + use_dfs : bool + If True, use DFS. In case some modules may have other options such as astroquery. dfs_node : str The environment in which the pipeline will run. Use "pml" for Purple Mountain Lab cluster and "local" for others. @@ -102,6 +102,8 @@ class CsstMsDataManager: The module log file name. clear_dir : bool If True, clear ``dm.dir_l1`` directory. + verbose : bool + If True, print verbose info. Examples -------- @@ -132,7 +134,7 @@ class CsstMsDataManager: dir_l0: str = ".", dir_l1: str = ".", path_aux: str = "", # bias dark flat - dfs_mode: bool = False, + use_dfs: bool = False, dfs_node: str = "kmust", obs_id: str = "100000100", exp_start: int = "20270810081950", @@ -145,6 +147,7 @@ class CsstMsDataManager: log_ppl="csst-l1ppl.log", log_mod="csst-l1mod.log", clear_dir=False, + verbose=True, ): # version @@ -166,14 +169,18 @@ class CsstMsDataManager: # ALL self.valid_detectors = CP["all"]["detectors"] self.detector2filter = CP["all"]["detector2filter"] - print("Data type is: ", self.datatype) - print("Valid detectors are: ", self.valid_detectors) + if verbose: + print("Data type is: ", self.datatype) + print("Valid detectors are: ", self.valid_detectors) # available_detectors self.available_detectors = available_detectors if available_detectors is not None else list() - print("Available detectors are:", self.available_detectors) + if verbose: + print("Available detectors are:", self.available_detectors) # set all available detectors by default self.target_detectors = target_detectors + if verbose: + print("Target detectors are: ", self._target_detectors) # exposure info self.obs_id = obs_id @@ -187,9 +194,14 @@ class CsstMsDataManager: self.obs_type = obs_type self.l0_post = l0_post - # for catalog query - self.dfs_mode = dfs_mode + # DFS configuration + self.use_dfs = use_dfs self.dfs_node = dfs_node + # DFS APIs + self.dfs_L0DataApi = Level0DataApi() + self.dfs_L1DataApi = Level1DataApi() + self.dfs_L0PrcApi = Level0PrcApi() + self.dfs_CalApi = CalMergeApi() # data directory self.dir_l0 = dir_l0 @@ -212,16 +224,12 @@ class CsstMsDataManager: # pipeline logger self.logger_ppl = get_logger(name="CSST L1 Pipeline Logger", filename=os.path.join(dir_l1, log_ppl)) - self.logger_ppl.info("logger_ppl initialized") + if verbose: + self.logger_ppl.info("logger_ppl initialized") # module logger self.logger_mod = get_logger(name="CSST L1 Module Logger", filename=os.path.join(dir_l1, log_mod)) - self.logger_mod.info("logger_mod initialized") - - # DFS APIs - self.dfs_L0DataApi = Level0DataApi() - self.dfs_L1DataApi = Level1DataApi() - self.dfs_L0PrcApi = Level0PrcApi() # level0到level1的处理过程记录 - self.dfs_CalApi = CalMergeApi() # 参考文件合并文件操作API + if verbose: + self.logger_mod.info("logger_mod initialized") def set_env(self): """ set environment variables """ @@ -245,7 +253,6 @@ class CsstMsDataManager: self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & set(detectors)) elif isinstance(detectors, int): self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & {detectors}) - print("Target detectors are: ", self._target_detectors) def set_detectors(self, detectors=None): raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!") @@ -281,7 +288,7 @@ class CsstMsDataManager: dir_l0=dir_l0, dir_l1=dir_l1, path_aux=path_aux, # bias dark flat - dfs_mode=dfs_mode, + use_dfs=dfs_mode, dfs_node=dfs_node, obs_id=obs_id, exp_start=exp_start, @@ -426,7 +433,7 @@ class CsstMsDataManager: def get_sls_info(self): """ Get the target SLS image header info and return. """ - # if self.dfs_mode: + # if self.use_dfs: # raise NotImplementedError() # else: assert len(self.target_detectors) == 1 @@ -435,7 +442,7 @@ class CsstMsDataManager: def get_mbi_info(self): """ Get all MBI image header info and return as a table. """ - # if self.dfs_mode: + # if self.use_dfs: # raise NotImplementedError() # else: info = Table.read("/nfsdata/share/csst_simulation_data/Cycle-5-SimuData/slitlessSpectroscopy/t_mbi_l1.fits") @@ -509,7 +516,7 @@ class CsstMsDataManager: lines += f"- Target detectors = {self.target_detectors}\n" lines += f"- dir_l0 = {self.dir_l0}\n" lines += f"- dir_l1 = {self.dir_l1}\n" - lines += f"- dfs_mode = {self.dfs_mode}\n" + lines += f"- use_dfs = {self.use_dfs}\n" lines += f"- dfs_node = {self.dfs_node}\n" lines += f"- CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n" return lines @@ -573,6 +580,7 @@ class CsstMsDataManager: def from_dfs(obs_id="100000100", datatype="mbi", dir_l0="/L1Pipeline/L0", dir_l1="/L1Pipeline/L1", dfs_mode=True, dfs_node="pml"): """ Initialize CsstMsDataManager from DFS. """ + print(f"Query obs_id={obs_id} ...", end="") records = CsstMsDataManager(dfs_node=dfs_node).dfs_L0DataApi.find(obs_id=obs_id) print(f"{records['totalCount']} records obtained!") tbl = Table([_.__dict__ for _ in records["data"]]) -- GitLab