diff --git a/csst_common/data/dfs_conf.yml b/csst_common/data/dfs_conf.yml index 397a25a9591e24cec010aaf178b8e87db04d553d..3d23eb4308c4297dd7709ad9575216f2cbaacc66 100644 --- a/csst_common/data/dfs_conf.yml +++ b/csst_common/data/dfs_conf.yml @@ -1,12 +1,12 @@ --- pml: - CSST_DFS_API_MODE: cluster - CSST_DFS_GATEWAY: 172.24.21.2:30880 - CSST_DFS_APP_ID: test - CSST_DFS_APP_TOKEN: test + CSST_DFS_API_MODE: "cluster" + CSST_DFS_GATEWAY: "172.24.21.2:30880" + CSST_DFS_APP_ID: "test" + CSST_DFS_APP_TOKEN: "test" kmust: - CSST_DFS_API_MODE: cluster - CSST_DFS_GATEWAY: 222.197.214.168:30880 - CSST_DFS_APP_ID: 1 - CSST_DFS_APP_TOKEN: 1 + CSST_DFS_API_MODE: "cluster" + CSST_DFS_GATEWAY: "222.197.214.168:30880" + CSST_DFS_APP_ID: "1" + CSST_DFS_APP_TOKEN: "1" diff --git a/csst_common/data_manager.py b/csst_common/data_manager.py index 1e4f652df28e62c437582625d47f31aed2a8d404..cbf98653820e877d7b01bc47c83d48369bf0eb51 100644 --- a/csst_common/data_manager.py +++ b/csst_common/data_manager.py @@ -149,12 +149,14 @@ class CsstMsDataManager: # ALL self.valid_detectors = CP["all"]["detectors"] self.detector2filter = CP["all"]["detector2filter"] + print("Data type is: ", self.datatype) + print("Valid detectors are: ", self.valid_detectors) # available_detectors - self.available_detectors = available_detectors + self.available_detectors = available_detectors if available_detectors is not None else list() + print("Available detectors are:", self.available_detectors) # set all available detectors by default - self.target_detectors = [] - self.set_detectors(target_detectors) + self.target_detectors = target_detectors # exposure info self._exp_id = _exp_id @@ -170,11 +172,7 @@ class CsstMsDataManager: # for catalog query self.dfs_mode = dfs_mode - # for DFS configuration, defaults to "local", could be "pml" - assert dfs_node in DFS_CONF.keys() - self.node = dfs_node - for k, v in DFS_CONF[dfs_node].items(): - os.putenv(k, f"{v}") + self.dfs_node = dfs_node # data directory self.dir_l0 = dir_l0 @@ -184,8 +182,38 @@ class CsstMsDataManager: # record hard code names in history self.hardcode_history = [] + @property + def dfs_node(self): + return self._dfs_node + + @dfs_node.setter + def dfs_node(self, dfs_node): + # for DFS configuration, defaults to "local", could be "pml" + assert dfs_node in DFS_CONF.keys() + self._dfs_node = dfs_node + for k, v in DFS_CONF[dfs_node].items(): + os.environ[k] = v + + @property + def target_detectors(self): + return self._target_detectors + + @target_detectors.setter + def target_detectors(self, detectors: Union[None, list, int] = None): + assert detectors is None or type(detectors) in [list, int] + if detectors is None: + self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors)) + elif isinstance(detectors, list): + self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & set(detectors)) + elif isinstance(detectors, int): + self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & {detectors}) + print("Target detectors are: ", self._target_detectors) + + def set_detectors(self, detectors=None): + raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!") + @staticmethod - def from_dfs(ver_sim="C5.2", data_type="mbi", exp_id=10000100, dir_l1="."): + def from_dfs(ver_sim="C5.2", datatype="mbi", exp_id=10000100, dir_l1="."): raise NotImplementedError("from_dfs is currently not available!") @staticmethod @@ -310,26 +338,6 @@ class CsstMsDataManager: self._exp_start, self._exp_stop, self._exp_id, detector, post) return os.path.join(self.dir_l1, fn) - def set_detectors(self, detectors=None): - """ set target detector """ - if detectors is None: - # default detectors - self.target_detectors = self.available_detectors - else: - try: - # assert detectors is a subset of available detectors - assert set(detectors).issubset(set(self.available_detectors)) - self.target_detectors = list(detectors) - except AssertionError as ae: - print("@DM: available detectors are ", self.available_detectors) - print("@DM: target detectors are ", detectors) - - print("@DM: final target detectors are ", set(detectors) & set(self.available_detectors)) - # raise ae - self.target_detectors = list(set(detectors) & set(self.available_detectors)) - print("final target detector IDs are ", self.target_detectors) - return - def get_bias(self, detector=6): """ get bias data """ if self.datatype == "mbi": @@ -440,7 +448,6 @@ class CsstMsDataManager: dir_l0 = "/share/simudata/CSSOSDataProductsSims/data/CSSTSimImage_C5/" \ "NGP_AstrometryON_shearOFF_Spec/MSC_{:07d}/".format(exposure_id) path_aux = "" - else: raise ValueError("@DM: invalid hostname {} or datatype {}!".format(hostname, datatype)) @@ -458,6 +465,20 @@ class CsstMsDataManager: except Exception as e: return None + def __repr__(self): + lines = "" + lines += "\n" + lines += f"Data type = {self.datatype}\n" + lines += f"Valid detectors = {self.valid_detectors}\n" + lines += f"Available detectors = {self.available_detectors}\n" + lines += f"Target detectors = {self.target_detectors}\n" + lines += f"dir_l0 = {self.dir_l0}\n" + lines += f"dir_l1 = {self.dir_l1}\n" + lines += f"dfs_mode = {self.dfs_mode}\n" + lines += f"dfs_node = {self.dfs_node}\n" + lines += f"CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n" + return lines + class CsstMbiDataManager: diff --git a/tests/test_data_manager.py b/tests/test_data_manager.py index d9e79cf61155ede5ee5ab0df39ed2318177755fe..9fc0faea08242a95e3768fe23e48e35133f49109 100644 --- a/tests/test_data_manager.py +++ b/tests/test_data_manager.py @@ -7,10 +7,11 @@ Created: 2022-09-13 Modified-History: 2022-09-13, Bo Zhang, created 2022-09-29, Bo Zhang, added test for CsstMbiDataManager + 2022-10-28, Bo Zhang, deleted unit test for CsstMsDataManager """ import os import unittest -from csst_common.data_manager import CsstMbiDataManager, CsstMsDataManager +from csst_common.data_manager import CsstMsDataManager from csst_common.params import CSST_PARAMS as CP