make dfs_node and target_detectors be properties (b68b1d88) · Commits · csst-pipeline / csst_common

csst_common/data/dfs_conf.yml

+8 −8

Original line number	Diff line number	Diff line
		---
		pml:
		CSST_DFS_API_MODE: cluster
		CSST_DFS_GATEWAY: 172.24.21.2:30880
		CSST_DFS_APP_ID: test
		CSST_DFS_APP_TOKEN: test
		CSST_DFS_API_MODE: "cluster"
		CSST_DFS_GATEWAY: "172.24.21.2:30880"
		CSST_DFS_APP_ID: "test"
		CSST_DFS_APP_TOKEN: "test"

		kmust:
		CSST_DFS_API_MODE: cluster
		CSST_DFS_GATEWAY: 222.197.214.168:30880
		CSST_DFS_APP_ID: 1
		CSST_DFS_APP_TOKEN: 1
		CSST_DFS_API_MODE: "cluster"
		CSST_DFS_GATEWAY: "222.197.214.168:30880"
		CSST_DFS_APP_ID: "1"
		CSST_DFS_APP_TOKEN: "1"

csst_common/data_manager.py

+51 −30

Original line number	Diff line number	Diff line
		@@ -149,12 +149,14 @@ class CsstMsDataManager:
		# ALL
		self.valid_detectors = CP["all"]["detectors"]
		self.detector2filter = CP["all"]["detector2filter"]
		print("Data type is: ", self.datatype)
		print("Valid detectors are: ", self.valid_detectors)

		# available_detectors
		self.available_detectors = available_detectors
		self.available_detectors = available_detectors if available_detectors is not None else list()
		print("Available detectors are:", self.available_detectors)
		# set all available detectors by default
		self.target_detectors = []
		self.set_detectors(target_detectors)
		self.target_detectors = target_detectors

		# exposure info
		self._exp_id = _exp_id
		@@ -170,11 +172,7 @@ class CsstMsDataManager:

		# for catalog query
		self.dfs_mode = dfs_mode
		# for DFS configuration, defaults to "local", could be "pml"
		assert dfs_node in DFS_CONF.keys()
		self.node = dfs_node
		for k, v in DFS_CONF[dfs_node].items():
		os.putenv(k, f"{v}")
		self.dfs_node = dfs_node

		# data directory
		self.dir_l0 = dir_l0
		@@ -184,8 +182,38 @@ class CsstMsDataManager:
		# record hard code names in history
		self.hardcode_history = []

		@property
		def dfs_node(self):
		return self._dfs_node

		@dfs_node.setter
		def dfs_node(self, dfs_node):
		# for DFS configuration, defaults to "local", could be "pml"
		assert dfs_node in DFS_CONF.keys()
		self._dfs_node = dfs_node
		for k, v in DFS_CONF[dfs_node].items():
		os.environ[k] = v

		@property
		def target_detectors(self):
		return self._target_detectors

		@target_detectors.setter
		def target_detectors(self, detectors: Union[None, list, int] = None):
		assert detectors is None or type(detectors) in [list, int]
		if detectors is None:
		self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors))
		elif isinstance(detectors, list):
		self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & set(detectors))
		elif isinstance(detectors, int):
		self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & {detectors})
		print("Target detectors are: ", self._target_detectors)

		def set_detectors(self, detectors=None):
		raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!")

		@staticmethod
		def from_dfs(ver_sim="C5.2", data_type="mbi", exp_id=10000100, dir_l1="."):
		def from_dfs(ver_sim="C5.2", datatype="mbi", exp_id=10000100, dir_l1="."):
		raise NotImplementedError("from_dfs is currently not available!")

		@staticmethod
		@@ -310,26 +338,6 @@ class CsstMsDataManager:
		self._exp_start, self._exp_stop, self._exp_id, detector, post)
		return os.path.join(self.dir_l1, fn)

		def set_detectors(self, detectors=None):
		""" set target detector """
		if detectors is None:
		# default detectors
		self.target_detectors = self.available_detectors
		else:
		try:
		# assert detectors is a subset of available detectors
		assert set(detectors).issubset(set(self.available_detectors))
		self.target_detectors = list(detectors)
		except AssertionError as ae:
		print("@DM: available detectors are ", self.available_detectors)
		print("@DM: target detectors are ", detectors)

		print("@DM: final target detectors are ", set(detectors) & set(self.available_detectors))
		# raise ae
		self.target_detectors = list(set(detectors) & set(self.available_detectors))
		print("final target detector IDs are ", self.target_detectors)
		return

		def get_bias(self, detector=6):
		""" get bias data """
		if self.datatype == "mbi":
		@@ -440,7 +448,6 @@ class CsstMsDataManager:
		dir_l0 = "/share/simudata/CSSOSDataProductsSims/data/CSSTSimImage_C5/" \
		"NGP_AstrometryON_shearOFF_Spec/MSC_{:07d}/".format(exposure_id)
		path_aux = ""

		else:
		raise ValueError("@DM: invalid hostname {} or datatype {}!".format(hostname, datatype))

		@@ -458,6 +465,20 @@ class CsstMsDataManager:
		except Exception as e:
		return None

		def __repr__(self):
		lines = ""
		lines += "<CsstMsDataManager>\n"
		lines += f"Data type = {self.datatype}\n"
		lines += f"Valid detectors = {self.valid_detectors}\n"
		lines += f"Available detectors = {self.available_detectors}\n"
		lines += f"Target detectors = {self.target_detectors}\n"
		lines += f"dir_l0 = {self.dir_l0}\n"
		lines += f"dir_l1 = {self.dir_l1}\n"
		lines += f"dfs_mode = {self.dfs_mode}\n"
		lines += f"dfs_node = {self.dfs_node}\n"
		lines += f"CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n"
		return lines


		class CsstMbiDataManager:

tests/test_data_manager.py

+2 −1

Original line number	Diff line number	Diff line
		@@ -7,10 +7,11 @@ Created: 2022-09-13
		Modified-History:
		2022-09-13, Bo Zhang, created
		2022-09-29, Bo Zhang, added test for CsstMbiDataManager
		2022-10-28, Bo Zhang, deleted unit test for CsstMsDataManager
		"""
		import os
		import unittest
		from csst_common.data_manager import CsstMbiDataManager, CsstMsDataManager
		from csst_common.data_manager import CsstMsDataManager
		from csst_common.params import CSST_PARAMS as CP