add verbose option (a6b8bf1a) · Commits · Wu Maochun / csst_common

csst_common/data_manager.py

+30 −22

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@ Modified-History:
		2022-09-13, Bo Zhang, added CsstMbiDataManager
		2022-09-29, Bo Zhang, favor CsstMsDataManager instead of CsstMbiDataManager
		2022-10-26, Bo Zhang, reconstruct CsstMsDataManager, deprecate CsstMbiDataManager
		2022-10-28, Bo Zhang, added CsstMsDataManager.query_rc(), dm.dfs_mode, dm.node
		2022-10-28, Bo Zhang, added CsstMsDataManager.query_rc(), dm.use_dfs, dm.node
		2022-11-06, Bo Zhang, deleted CsstMbiDataManager
		2022-11-20, Bo Zhang, added DFS APIs
		"""
		@@ -75,8 +75,8 @@ class CsstMsDataManager:
		The L1 directory.
		path_aux : str
		The aux data directory (bias, flat, dark).
		dfs_mode : bool
		DFS mode. If True, ``CsstMsDataManager`` will use DFS.
		use_dfs : bool
		If True, use DFS. In case some modules may have other options such as astroquery.
		dfs_node : str
		The environment in which the pipeline will run.
		Use "pml" for Purple Mountain Lab cluster and "local" for others.
		@@ -102,6 +102,8 @@ class CsstMsDataManager:
		The module log file name.
		clear_dir : bool
		If True, clear ``dm.dir_l1`` directory.
		verbose : bool
		If True, print verbose info.

		Examples
		--------
		@@ -132,7 +134,7 @@ class CsstMsDataManager:
		dir_l0: str = ".",
		dir_l1: str = ".",
		path_aux: str = "", # bias dark flat
		dfs_mode: bool = False,
		use_dfs: bool = False,
		dfs_node: str = "kmust",
		obs_id: str = "100000100",
		exp_start: int = "20270810081950",
		@@ -145,6 +147,7 @@ class CsstMsDataManager:
		log_ppl="csst-l1ppl.log",
		log_mod="csst-l1mod.log",
		clear_dir=False,
		verbose=True,
		):

		# version
		@@ -166,14 +169,18 @@ class CsstMsDataManager:
		# ALL
		self.valid_detectors = CP["all"]["detectors"]
		self.detector2filter = CP["all"]["detector2filter"]
		if verbose:
		print("Data type is: ", self.datatype)
		print("Valid detectors are: ", self.valid_detectors)

		# available_detectors
		self.available_detectors = available_detectors if available_detectors is not None else list()
		if verbose:
		print("Available detectors are:", self.available_detectors)
		# set all available detectors by default
		self.target_detectors = target_detectors
		if verbose:
		print("Target detectors are: ", self._target_detectors)

		# exposure info
		self.obs_id = obs_id
		@@ -187,9 +194,14 @@ class CsstMsDataManager:
		self.obs_type = obs_type
		self.l0_post = l0_post

		# for catalog query
		self.dfs_mode = dfs_mode
		# DFS configuration
		self.use_dfs = use_dfs
		self.dfs_node = dfs_node
		# DFS APIs
		self.dfs_L0DataApi = Level0DataApi()
		self.dfs_L1DataApi = Level1DataApi()
		self.dfs_L0PrcApi = Level0PrcApi()
		self.dfs_CalApi = CalMergeApi()

		# data directory
		self.dir_l0 = dir_l0
		@@ -212,17 +224,13 @@ class CsstMsDataManager:

		# pipeline logger
		self.logger_ppl = get_logger(name="CSST L1 Pipeline Logger", filename=os.path.join(dir_l1, log_ppl))
		if verbose:
		self.logger_ppl.info("logger_ppl initialized")
		# module logger
		self.logger_mod = get_logger(name="CSST L1 Module Logger", filename=os.path.join(dir_l1, log_mod))
		if verbose:
		self.logger_mod.info("logger_mod initialized")

		# DFS APIs
		self.dfs_L0DataApi = Level0DataApi()
		self.dfs_L1DataApi = Level1DataApi()
		self.dfs_L0PrcApi = Level0PrcApi() # level0到level1的处理过程记录
		self.dfs_CalApi = CalMergeApi() # 参考文件合并文件操作API

		def set_env(self):
		""" set environment variables """
		if os.uname()[1] == "dandelion":
		@@ -245,7 +253,6 @@ class CsstMsDataManager:
		self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & set(detectors))
		elif isinstance(detectors, int):
		self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & {detectors})
		print("Target detectors are: ", self._target_detectors)

		def set_detectors(self, detectors=None):
		raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!")
		@@ -281,7 +288,7 @@ class CsstMsDataManager:
		dir_l0=dir_l0,
		dir_l1=dir_l1,
		path_aux=path_aux, # bias dark flat
		dfs_mode=dfs_mode,
		use_dfs=dfs_mode,
		dfs_node=dfs_node,
		obs_id=obs_id,
		exp_start=exp_start,
		@@ -426,7 +433,7 @@ class CsstMsDataManager:

		def get_sls_info(self):
		""" Get the target SLS image header info and return. """
		# if self.dfs_mode:
		# if self.use_dfs:
		# raise NotImplementedError()
		# else:
		assert len(self.target_detectors) == 1
		@@ -435,7 +442,7 @@ class CsstMsDataManager:

		def get_mbi_info(self):
		""" Get all MBI image header info and return as a table. """
		# if self.dfs_mode:
		# if self.use_dfs:
		# raise NotImplementedError()
		# else:
		info = Table.read("/nfsdata/share/csst_simulation_data/Cycle-5-SimuData/slitlessSpectroscopy/t_mbi_l1.fits")
		@@ -509,7 +516,7 @@ class CsstMsDataManager:
		lines += f"- Target detectors = {self.target_detectors}\n"
		lines += f"- dir_l0 = {self.dir_l0}\n"
		lines += f"- dir_l1 = {self.dir_l1}\n"
		lines += f"- dfs_mode = {self.dfs_mode}\n"
		lines += f"- use_dfs = {self.use_dfs}\n"
		lines += f"- dfs_node = {self.dfs_node}\n"
		lines += f"- CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n"
		return lines
		@@ -573,6 +580,7 @@ class CsstMsDataManager:
		def from_dfs(obs_id="100000100", datatype="mbi", dir_l0="/L1Pipeline/L0", dir_l1="/L1Pipeline/L1",
		dfs_mode=True, dfs_node="pml"):
		""" Initialize CsstMsDataManager from DFS. """
		print(f"Query obs_id={obs_id} ...", end="")
		records = CsstMsDataManager(dfs_node=dfs_node).dfs_L0DataApi.find(obs_id=obs_id)
		print(f"{records['totalCount']} records obtained!")
		tbl = Table([_.__dict__ for _ in records["data"]])