From b3dd28370d68b0a0695346b75d0e3ec83ed1ef95 Mon Sep 17 00:00:00 2001 From: BO ZHANG Date: Mon, 9 Jan 2023 17:07:35 +0800 Subject: [PATCH] compatible with CRDS ref files --- csst_common/data_manager.py | 49 +++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/csst_common/data_manager.py b/csst_common/data_manager.py index cdf394d..d59f75c 100644 --- a/csst_common/data_manager.py +++ b/csst_common/data_manager.py @@ -140,7 +140,7 @@ class CsstMsDataManager: target_detectors: Union[None, list, int] = None, dir_l0: str = ".", dir_l1: str = ".", - path_aux: str = "", # bias dark flat + path_aux: str = "/L1Pipeline/aux", # aux dir use_dfs: bool = False, dfs_node: str = "kmust", dfs_root: str = "/share/dfs", @@ -213,6 +213,7 @@ class CsstMsDataManager: self.dir_l0 = dir_l0 self.dir_l1 = dir_l1 self.path_aux = path_aux + self.ref_version = None # record hard code names in history self.hardcode_history = [] @@ -435,39 +436,44 @@ class CsstMsDataManager: def get_bias(self, detector=6): """ get bias data """ if self.custom_bias is None: - if self.datatype == "mbi": - return glob.glob(self.path_aux.format("CLB", detector))[0] - else: - return self.path_aux.format(detector, "bias") + return os.path.join( + self.path_aux, + "C6.1_ref_crds", + "csst_msc_{}_{:02d}_{:04d}.fits".format( + "bias", detector, 1 if self.ref_version is None else self.ref_version + ) + ) else: return self.custom_bias.format("bias", detector) def get_dark(self, detector=6): """ get dark data """ if self.custom_dark is None: - if self.datatype == "mbi": - return glob.glob(self.path_aux.format("CLD", detector))[0] - else: - return self.path_aux.format(detector, "dark") + return os.path.join( + self.path_aux, + "C6.1_ref_crds", + "csst_msc_{}_{:02d}_{:04d}.fits".format( + "dark", detector, 1 if self.ref_version is None else self.ref_version + ) + ) else: return self.custom_dark.format("dark", detector) def get_flat(self, detector=6): """ get flat data """ if self.custom_flat is None: - if self.datatype == "mbi": - return glob.glob(self.path_aux.format("CLF", detector))[0] - else: - return self.path_aux.format(detector, "flat") + return os.path.join( + self.path_aux, + "C6.1_ref_crds", + "csst_msc_{}_{:02d}_{:04d}.fits".format( + "flat", detector, 1 if self.ref_version is None else self.ref_version + ) + ) else: return self.custom_flat.format("flat", detector) def get_axeconf(self): - if os.uname()[1] == "dandelion": - return "/home/csstpipeline/L1Pipeline/aux/axeconf" - else: - # in docker - return "/L1Pipeline/aux/axeconf" + return os.path.join(self.path_aux, "axeconf") # "/home/csstpipeline/L1Pipeline/aux/axeconf" def l1_file(self, name="", comment=""): """ L1 file path @@ -548,11 +554,11 @@ class CsstMsDataManager: if hostname == "dandelion" and datatype == "mbi": dir_l0 = "/nfsdata/share/csst_simulation_data/Cycle-5-SimuData/multipleBandsImaging/" \ "NGP_AstrometryON_shearOFF/MSC_{:07d}/".format(exposure_id) - path_aux = "/nfsdata/users/cham/L1Test/ref_C5.2/MSC_{}_*_{:02d}_combine.fits" + path_aux = "/nfsdata/users/csstpipeline/L1Pipeline/aux" elif hostname == "dandelion" and datatype == "sls": dir_l0 = "/nfsdata/share/csst_simulation_data/Cycle-5-SimuData/slitlessSpectroscopy/" \ "NGP_AstrometryON_shearOFF_Spec/MSC_{:07d}/".format(exposure_id) - path_aux = "/nfsdata/share/csst_simulation_data/Cycle-5-SimuData/slitlessSpectroscopy/csst_{:02d}{}.fits" + path_aux = "/nfsdata/users/csstpipeline/L1Pipeline/aux" # PMO elif hostname == "ubuntu" and datatype == "mbi": @@ -568,7 +574,8 @@ class CsstMsDataManager: return CsstMsDataManager.from_dir( ver_sim=ver_sim, datatype=datatype, dir_l0=dir_l0, dir_l1=dir_l1, path_aux=path_aux, - use_dfs=use_dfs, dfs_node=dfs_node, n_jobs=n_jobs, backend=backend) + use_dfs=use_dfs, dfs_node=dfs_node, n_jobs=n_jobs, backend=backend + ) def __repr__(self): lines = "" -- GitLab