From 33c2304a930442ddc6a038f31f9c20e6cf4dfa3f Mon Sep 17 00:00:00 2001 From: BO ZHANG Date: Fri, 18 Aug 2023 14:21:50 +0800 Subject: [PATCH] remove dfs note settings from CsstMsDataManager --- csst_common/data_manager.py | 76 +++++++------------------------------ 1 file changed, 13 insertions(+), 63 deletions(-) diff --git a/csst_common/data_manager.py b/csst_common/data_manager.py index 3e601e4..f5fd7b9 100644 --- a/csst_common/data_manager.py +++ b/csst_common/data_manager.py @@ -86,11 +86,6 @@ class CsstMsDataManager: The L1 directory. path_aux : str The aux data directory (bias, flat, dark). - use_dfs : bool - If True, use DFS. In case some modules may have other options such as astroquery. - dfs_node : str - The environment in which the pipeline will run. - Use "pml" for Purple Mountain Lab cluster and "local" for others. dfs_root : str The DFS root path. telescope : str @@ -160,10 +155,8 @@ class CsstMsDataManager: target_detectors: Union[None, list, int] = None, dir_l0: str = ".", dir_l1: str = ".", - path_aux: str = "/L1Pipeline/aux", # aux dir - use_dfs: bool = False, - dfs_node: str = "kmust", - dfs_root: str = "/share/dfs", + path_aux: str = "/pipeline/aux", # aux dir + dfs_root: str = "/dfsroot", telescope: str = "CSST", instrument: str = "MSC", project: str = "MS", @@ -235,8 +228,8 @@ class CsstMsDataManager: self.ext = ext # DFS configuration - self.use_dfs = use_dfs - self.dfs_node = dfs_node + # self.use_dfs = use_dfs + # self.dfs_node = dfs_node self.dfs_root = dfs_root # data directory @@ -338,15 +331,6 @@ class CsstMsDataManager: def dfs_CatApi(self): return CatalogApi() - def set_env(self): - """ set environment variables """ - if os.uname()[1] == "dandelion": - os.environ["LD_LIBRARY_PATH"] = "/home/csstpipeline/anaconda3/lib" - os.environ["AXE_BINDIR"] = "/home/csstpipeline/PycharmProjects/axe/cextern/src" - else: - os.environ["LD_LIBRARY_PATH"] = "" - os.environ["AXE_BINDIR"] = "" - @property def target_detectors(self): return self._target_detectors @@ -372,8 +356,6 @@ class CsstMsDataManager: dir_l1=".", path_aux="", pattern="CSST_MSC_*_SCIE_*.fits", - use_dfs=False, - dfs_node="kmust", log_ppl="csst-l1ppl.log", log_mod="csst-l1mod.log", n_jobs=18, @@ -440,8 +422,6 @@ class CsstMsDataManager: dir_l0=dir_l0, dir_l1=dir_l1, path_aux=path_aux, # bias dark flat - use_dfs=use_dfs, - dfs_node=dfs_node, log_ppl=log_ppl, log_mod=log_mod, n_jobs=n_jobs, @@ -529,19 +509,11 @@ class CsstMsDataManager: assert detector in self.available_detectors if post == ".fits": # no additional suffix - if self.ver_sim in ["C5.2", "C6.1"]: - fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \ - f"{self.obs_id}_{detector:02d}{post}" - else: - fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \ - f"{self.obs_id}_{detector:02d}_L1_V{self.version}{post}" + fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L1_V{self.version}{post}") else: - if self.ver_sim in ["C5.2", "C6.1"]: - fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \ - f"{self.obs_id}_{detector:02d}_{post}" - else: - fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \ - f"{self.obs_id}_{detector:02d}_L1_V{self.version}_{post}" + fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L1_V{self.version}_{post}") return os.path.join(self.dir_l1, fn) def get_refs(self, detector=6): @@ -730,7 +702,6 @@ class CsstMsDataManager: lines += f"- Target detectors = {self.target_detectors}\n" lines += f"- dir_l0 = {self.dir_l0}\n" lines += f"- dir_l1 = {self.dir_l1}\n" - lines += f"- use_dfs = {self.use_dfs}\n" lines += f"- dfs_node = {self.dfs_node}\n" lines += f"- CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n" return lines @@ -752,21 +723,6 @@ class CsstMsDataManager: def dfs_node(self): return self._dfs_node - @dfs_node.setter - def dfs_node(self, dfs_node): - # for DFS configuration, defaults to "local", could be "pml" - assert dfs_node in DFS_CONF.keys() - self._dfs_node = dfs_node - for k, v in DFS_CONF[dfs_node].items(): - os.environ[k] = v - - @staticmethod - def dfs_setup(dfs_node="pml"): - """ Setup DFS configuration. """ - assert dfs_node in DFS_CONF.keys() - for k, v in DFS_CONF[dfs_node].items(): - os.environ[k] = v - @staticmethod def dfs_is_available(): """ Test if DFS works. """ @@ -938,7 +894,6 @@ class CsstMsDataManager: dir_l1="/L1Pipeline/L1", path_aux="/L1Pipeline/aux", use_dfs=True, - dfs_node="pml", clear_l1=False, dfs_root="/share/dfs", n_jobs=18, @@ -964,7 +919,7 @@ class CsstMsDataManager: os.chdir(dir_l1) print(f"Query obs_id={obs_id} ...", end="") - records = CsstMsDataManager(dfs_node=dfs_node, verbose=False).dfs_L0DataApi.find(obs_id=obs_id) + records = CsstMsDataManager(verbose=False).dfs_L0DataApi.find(obs_id=obs_id) print(f"{records['totalCount']} records obtained!") tbl = Table([_.__dict__ for _ in records["data"]]) tbl.sort(["detector_no", "obs_type"]) @@ -984,7 +939,6 @@ class CsstMsDataManager: dir_l1=dir_l1, path_aux=path_aux, use_dfs=use_dfs, - dfs_node=dfs_node, n_jobs=n_jobs, backend=backend, device=device, @@ -1035,10 +989,6 @@ class CsstMsDataManager: pass -# temporarily compatible with old interface -CsstMbiDataManager = CsstMsDataManager - - class CsstMsFile(dict): def __init__(self, filepath, ver_sim="C6.2", dir_out=".", **kwargs): super(CsstMsFile, self).__init__() @@ -1090,11 +1040,11 @@ class CsstMsFile(dict): def fno(self, post=".fits"): if post.startswith("."): - fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \ - f"{self.obs_id}_{self.detector}_L1_V{self.version}{post}" + fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{self.detector}_L1_V{self.version}{post}") else: - fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \ - f"{self.obs_id}_{self.detector}_L1_V{self.version}_{post}" + fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{self.detector}_L1_V{self.version}_{post}") return fn def fpi(self): -- GitLab