Loading csst_common/data_manager.py +41 −4 Original line number Diff line number Diff line Loading @@ -252,10 +252,6 @@ class CsstMsDataManager: def set_detectors(self, detectors=None): raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!") @staticmethod def from_dfs(expid, ver_sim="C5.2", datatype="mbi", dir_l1="."): raise NotImplementedError("from_dfs is currently not available!") @staticmethod def from_dir(ver_sim="C5.2", datatype="mbi", dir_l0=".", dir_l1=".", path_aux="", dfs_mode=False, dfs_node="kmust"): """ initialize the multi-band imaging data manager """ Loading Loading @@ -571,6 +567,47 @@ class CsstMsDataManager: """ Push SLS spectra to DFS. """ pass @staticmethod def from_dfs(obs_id="100000100", datatype="mbi", dir_l0="/L1Pipeline/L0", dir_l1=".", dfs_mode=True, dfs_node="pml"): """ Initialize CsstMsDataManager from DFS. """ records = Level0DataApi().find(obs_id=obs_id) # query DFS tbl = Table([_.__dict__ for _ in records["data"]]) tbl.sort(["detector_no", "obs_type"]) if os.uname()[1] == "ubuntu": # use pml test node dfs_root = "/share/dfs" else: # in docker dfs_root = "/dfsroot" # (clear and) make directories if os.path.exists(dir_l0): os.system(f"rm -rf {dir_l0}") if os.path.exists(dir_l1): os.system(f"rm -rf {dir_l1}") os.mkdir(dir_l0) os.mkdir(dir_l1) for i_rec in range(len(tbl)): # make symbolic links os.symlink( src=os.path.join(dfs_root, tbl["file_path"][i_rec]), dst=os.path.join(dir_l0, tbl["file_path"][i_rec]) ) if datatype == "mbi": path_aux = "/share/L1Pipeline/aux/C5.2_ref_mbi/MSC_{}_*_{:02d}_combine.fits" elif datatype == "sls": path_aux = "/share/L1Pipeline/aux/C5.2_ref_sls/csst_{:02d}{}.fits" else: path_aux = "" # initialize dm dm = CsstMsDataManager.from_dir( ver_sim="C5.2", datatype=datatype, dir_l0=dir_l0, dir_l1=dir_l1, path_aux=path_aux, dfs_mode=dfs_mode, dfs_node=dfs_node ) return dm def dfs_l0_query(self, obs_id: str = "100000100"): """ Query L0 data from DFS. """ result = self.dfs_L0DataApi.find(obs_id=str(obs_id)) Loading Loading
csst_common/data_manager.py +41 −4 Original line number Diff line number Diff line Loading @@ -252,10 +252,6 @@ class CsstMsDataManager: def set_detectors(self, detectors=None): raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!") @staticmethod def from_dfs(expid, ver_sim="C5.2", datatype="mbi", dir_l1="."): raise NotImplementedError("from_dfs is currently not available!") @staticmethod def from_dir(ver_sim="C5.2", datatype="mbi", dir_l0=".", dir_l1=".", path_aux="", dfs_mode=False, dfs_node="kmust"): """ initialize the multi-band imaging data manager """ Loading Loading @@ -571,6 +567,47 @@ class CsstMsDataManager: """ Push SLS spectra to DFS. """ pass @staticmethod def from_dfs(obs_id="100000100", datatype="mbi", dir_l0="/L1Pipeline/L0", dir_l1=".", dfs_mode=True, dfs_node="pml"): """ Initialize CsstMsDataManager from DFS. """ records = Level0DataApi().find(obs_id=obs_id) # query DFS tbl = Table([_.__dict__ for _ in records["data"]]) tbl.sort(["detector_no", "obs_type"]) if os.uname()[1] == "ubuntu": # use pml test node dfs_root = "/share/dfs" else: # in docker dfs_root = "/dfsroot" # (clear and) make directories if os.path.exists(dir_l0): os.system(f"rm -rf {dir_l0}") if os.path.exists(dir_l1): os.system(f"rm -rf {dir_l1}") os.mkdir(dir_l0) os.mkdir(dir_l1) for i_rec in range(len(tbl)): # make symbolic links os.symlink( src=os.path.join(dfs_root, tbl["file_path"][i_rec]), dst=os.path.join(dir_l0, tbl["file_path"][i_rec]) ) if datatype == "mbi": path_aux = "/share/L1Pipeline/aux/C5.2_ref_mbi/MSC_{}_*_{:02d}_combine.fits" elif datatype == "sls": path_aux = "/share/L1Pipeline/aux/C5.2_ref_sls/csst_{:02d}{}.fits" else: path_aux = "" # initialize dm dm = CsstMsDataManager.from_dir( ver_sim="C5.2", datatype=datatype, dir_l0=dir_l0, dir_l1=dir_l1, path_aux=path_aux, dfs_mode=dfs_mode, dfs_node=dfs_node ) return dm def dfs_l0_query(self, obs_id: str = "100000100"): """ Query L0 data from DFS. """ result = self.dfs_L0DataApi.find(obs_id=str(obs_id)) Loading