add verbose option (a6b8bf1a) · Commits · csst-pipeline / csst_common

csst_common/data_manager.py

+30 −22

Original line number	Original line	Diff line number	Diff line
	@@ -9,7 +9,7 @@ Modified-History:
	2022-09-13, Bo Zhang, added CsstMbiDataManager		2022-09-13, Bo Zhang, added CsstMbiDataManager
	2022-09-29, Bo Zhang, favor CsstMsDataManager instead of CsstMbiDataManager		2022-09-29, Bo Zhang, favor CsstMsDataManager instead of CsstMbiDataManager
	2022-10-26, Bo Zhang, reconstruct CsstMsDataManager, deprecate CsstMbiDataManager		2022-10-26, Bo Zhang, reconstruct CsstMsDataManager, deprecate CsstMbiDataManager
	2022-10-28, Bo Zhang, added CsstMsDataManager.query_rc(), dm.dfs_mode, dm.node		2022-10-28, Bo Zhang, added CsstMsDataManager.query_rc(), dm.use_dfs, dm.node
	2022-11-06, Bo Zhang, deleted CsstMbiDataManager		2022-11-06, Bo Zhang, deleted CsstMbiDataManager
	2022-11-20, Bo Zhang, added DFS APIs		2022-11-20, Bo Zhang, added DFS APIs
	"""		"""
	@@ -75,8 +75,8 @@ class CsstMsDataManager:
	The L1 directory.		The L1 directory.
	path_aux : str		path_aux : str
	The aux data directory (bias, flat, dark).		The aux data directory (bias, flat, dark).
	dfs_mode : bool		use_dfs : bool
	DFS mode. If True, ``CsstMsDataManager`` will use DFS.		If True, use DFS. In case some modules may have other options such as astroquery.
	dfs_node : str		dfs_node : str
	The environment in which the pipeline will run.		The environment in which the pipeline will run.
	Use "pml" for Purple Mountain Lab cluster and "local" for others.		Use "pml" for Purple Mountain Lab cluster and "local" for others.
	@@ -102,6 +102,8 @@ class CsstMsDataManager:
	The module log file name.		The module log file name.
	clear_dir : bool		clear_dir : bool
	If True, clear ``dm.dir_l1`` directory.		If True, clear ``dm.dir_l1`` directory.
			verbose : bool
			If True, print verbose info.

	Examples		Examples
	--------		--------
	@@ -132,7 +134,7 @@ class CsstMsDataManager:
	dir_l0: str = ".",		dir_l0: str = ".",
	dir_l1: str = ".",		dir_l1: str = ".",
	path_aux: str = "", # bias dark flat		path_aux: str = "", # bias dark flat
	dfs_mode: bool = False,		use_dfs: bool = False,
	dfs_node: str = "kmust",		dfs_node: str = "kmust",
	obs_id: str = "100000100",		obs_id: str = "100000100",
	exp_start: int = "20270810081950",		exp_start: int = "20270810081950",
	@@ -145,6 +147,7 @@ class CsstMsDataManager:
	log_ppl="csst-l1ppl.log",		log_ppl="csst-l1ppl.log",
	log_mod="csst-l1mod.log",		log_mod="csst-l1mod.log",
	clear_dir=False,		clear_dir=False,
			verbose=True,
	):		):

	# version		# version
	@@ -166,14 +169,18 @@ class CsstMsDataManager:
	# ALL		# ALL
	self.valid_detectors = CP["all"]["detectors"]		self.valid_detectors = CP["all"]["detectors"]
	self.detector2filter = CP["all"]["detector2filter"]		self.detector2filter = CP["all"]["detector2filter"]
			if verbose:
	print("Data type is: ", self.datatype)		print("Data type is: ", self.datatype)
	print("Valid detectors are: ", self.valid_detectors)		print("Valid detectors are: ", self.valid_detectors)

	# available_detectors		# available_detectors
	self.available_detectors = available_detectors if available_detectors is not None else list()		self.available_detectors = available_detectors if available_detectors is not None else list()
			if verbose:
	print("Available detectors are:", self.available_detectors)		print("Available detectors are:", self.available_detectors)
	# set all available detectors by default		# set all available detectors by default
	self.target_detectors = target_detectors		self.target_detectors = target_detectors
			if verbose:
			print("Target detectors are: ", self._target_detectors)

	# exposure info		# exposure info
	self.obs_id = obs_id		self.obs_id = obs_id
	@@ -187,9 +194,14 @@ class CsstMsDataManager:
	self.obs_type = obs_type		self.obs_type = obs_type
	self.l0_post = l0_post		self.l0_post = l0_post

	# for catalog query		# DFS configuration
	self.dfs_mode = dfs_mode		self.use_dfs = use_dfs
	self.dfs_node = dfs_node		self.dfs_node = dfs_node
			# DFS APIs
			self.dfs_L0DataApi = Level0DataApi()
			self.dfs_L1DataApi = Level1DataApi()
			self.dfs_L0PrcApi = Level0PrcApi()
			self.dfs_CalApi = CalMergeApi()

	# data directory		# data directory
	self.dir_l0 = dir_l0		self.dir_l0 = dir_l0
	@@ -212,17 +224,13 @@ class CsstMsDataManager:

	# pipeline logger		# pipeline logger
	self.logger_ppl = get_logger(name="CSST L1 Pipeline Logger", filename=os.path.join(dir_l1, log_ppl))		self.logger_ppl = get_logger(name="CSST L1 Pipeline Logger", filename=os.path.join(dir_l1, log_ppl))
			if verbose:
	self.logger_ppl.info("logger_ppl initialized")		self.logger_ppl.info("logger_ppl initialized")
	# module logger		# module logger
	self.logger_mod = get_logger(name="CSST L1 Module Logger", filename=os.path.join(dir_l1, log_mod))		self.logger_mod = get_logger(name="CSST L1 Module Logger", filename=os.path.join(dir_l1, log_mod))
			if verbose:
	self.logger_mod.info("logger_mod initialized")		self.logger_mod.info("logger_mod initialized")

	# DFS APIs
	self.dfs_L0DataApi = Level0DataApi()
	self.dfs_L1DataApi = Level1DataApi()
	self.dfs_L0PrcApi = Level0PrcApi() # level0到level1的处理过程记录
	self.dfs_CalApi = CalMergeApi() # 参考文件合并文件操作API

	def set_env(self):		def set_env(self):
	""" set environment variables """		""" set environment variables """
	if os.uname()[1] == "dandelion":		if os.uname()[1] == "dandelion":
	@@ -245,7 +253,6 @@ class CsstMsDataManager:
	self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & set(detectors))		self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & set(detectors))
	elif isinstance(detectors, int):		elif isinstance(detectors, int):
	self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & {detectors})		self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & {detectors})
	print("Target detectors are: ", self._target_detectors)

	def set_detectors(self, detectors=None):		def set_detectors(self, detectors=None):
	raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!")		raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!")
	@@ -281,7 +288,7 @@ class CsstMsDataManager:
	dir_l0=dir_l0,		dir_l0=dir_l0,
	dir_l1=dir_l1,		dir_l1=dir_l1,
	path_aux=path_aux, # bias dark flat		path_aux=path_aux, # bias dark flat
	dfs_mode=dfs_mode,		use_dfs=dfs_mode,
	dfs_node=dfs_node,		dfs_node=dfs_node,
	obs_id=obs_id,		obs_id=obs_id,
	exp_start=exp_start,		exp_start=exp_start,
	@@ -426,7 +433,7 @@ class CsstMsDataManager:

	def get_sls_info(self):		def get_sls_info(self):
	""" Get the target SLS image header info and return. """		""" Get the target SLS image header info and return. """
	# if self.dfs_mode:		# if self.use_dfs:
	# raise NotImplementedError()		# raise NotImplementedError()
	# else:		# else:
	assert len(self.target_detectors) == 1		assert len(self.target_detectors) == 1
	@@ -435,7 +442,7 @@ class CsstMsDataManager:

	def get_mbi_info(self):		def get_mbi_info(self):
	""" Get all MBI image header info and return as a table. """		""" Get all MBI image header info and return as a table. """
	# if self.dfs_mode:		# if self.use_dfs:
	# raise NotImplementedError()		# raise NotImplementedError()
	# else:		# else:
	info = Table.read("/nfsdata/share/csst_simulation_data/Cycle-5-SimuData/slitlessSpectroscopy/t_mbi_l1.fits")		info = Table.read("/nfsdata/share/csst_simulation_data/Cycle-5-SimuData/slitlessSpectroscopy/t_mbi_l1.fits")
	@@ -509,7 +516,7 @@ class CsstMsDataManager:
	lines += f"- Target detectors = {self.target_detectors}\n"		lines += f"- Target detectors = {self.target_detectors}\n"
	lines += f"- dir_l0 = {self.dir_l0}\n"		lines += f"- dir_l0 = {self.dir_l0}\n"
	lines += f"- dir_l1 = {self.dir_l1}\n"		lines += f"- dir_l1 = {self.dir_l1}\n"
	lines += f"- dfs_mode = {self.dfs_mode}\n"		lines += f"- use_dfs = {self.use_dfs}\n"
	lines += f"- dfs_node = {self.dfs_node}\n"		lines += f"- dfs_node = {self.dfs_node}\n"
	lines += f"- CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n"		lines += f"- CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n"
	return lines		return lines
	@@ -573,6 +580,7 @@ class CsstMsDataManager:
	def from_dfs(obs_id="100000100", datatype="mbi", dir_l0="/L1Pipeline/L0", dir_l1="/L1Pipeline/L1",		def from_dfs(obs_id="100000100", datatype="mbi", dir_l0="/L1Pipeline/L0", dir_l1="/L1Pipeline/L1",
	dfs_mode=True, dfs_node="pml"):		dfs_mode=True, dfs_node="pml"):
	""" Initialize CsstMsDataManager from DFS. """		""" Initialize CsstMsDataManager from DFS. """
			print(f"Query obs_id={obs_id} ...", end="")
	records = CsstMsDataManager(dfs_node=dfs_node).dfs_L0DataApi.find(obs_id=obs_id)		records = CsstMsDataManager(dfs_node=dfs_node).dfs_L0DataApi.find(obs_id=obs_id)
	print(f"{records['totalCount']} records obtained!")		print(f"{records['totalCount']} records obtained!")
	tbl = Table([_.__dict__ for _ in records["data"]])		tbl = Table([_.__dict__ for _ in records["data"]])