remove dfs note settings from CsstMsDataManager (33c2304a) · Commits · csst-pipeline / csst_common

csst_common/data_manager.py

+13 −63

Original line number	Diff line number	Diff line
		@@ -86,11 +86,6 @@ class CsstMsDataManager:
		The L1 directory.
		path_aux : str
		The aux data directory (bias, flat, dark).
		use_dfs : bool
		If True, use DFS. In case some modules may have other options such as astroquery.
		dfs_node : str
		The environment in which the pipeline will run.
		Use "pml" for Purple Mountain Lab cluster and "local" for others.
		dfs_root : str
		The DFS root path.
		telescope : str
		@@ -160,10 +155,8 @@ class CsstMsDataManager:
		target_detectors: Union[None, list, int] = None,
		dir_l0: str = ".",
		dir_l1: str = ".",
		path_aux: str = "/L1Pipeline/aux", # aux dir
		use_dfs: bool = False,
		dfs_node: str = "kmust",
		dfs_root: str = "/share/dfs",
		path_aux: str = "/pipeline/aux", # aux dir
		dfs_root: str = "/dfsroot",
		telescope: str = "CSST",
		instrument: str = "MSC",
		project: str = "MS",
		@@ -235,8 +228,8 @@ class CsstMsDataManager:
		self.ext = ext

		# DFS configuration
		self.use_dfs = use_dfs
		self.dfs_node = dfs_node
		# self.use_dfs = use_dfs
		# self.dfs_node = dfs_node
		self.dfs_root = dfs_root

		# data directory
		@@ -338,15 +331,6 @@ class CsstMsDataManager:
		def dfs_CatApi(self):
		return CatalogApi()

		def set_env(self):
		""" set environment variables """
		if os.uname()[1] == "dandelion":
		os.environ["LD_LIBRARY_PATH"] = "/home/csstpipeline/anaconda3/lib"
		os.environ["AXE_BINDIR"] = "/home/csstpipeline/PycharmProjects/axe/cextern/src"
		else:
		os.environ["LD_LIBRARY_PATH"] = ""
		os.environ["AXE_BINDIR"] = ""

		@property
		def target_detectors(self):
		return self._target_detectors
		@@ -372,8 +356,6 @@ class CsstMsDataManager:
		dir_l1=".",
		path_aux="",
		pattern="CSST_MSC__SCIE_.fits",
		use_dfs=False,
		dfs_node="kmust",
		log_ppl="csst-l1ppl.log",
		log_mod="csst-l1mod.log",
		n_jobs=18,
		@@ -440,8 +422,6 @@ class CsstMsDataManager:
		dir_l0=dir_l0,
		dir_l1=dir_l1,
		path_aux=path_aux, # bias dark flat
		use_dfs=use_dfs,
		dfs_node=dfs_node,
		log_ppl=log_ppl,
		log_mod=log_mod,
		n_jobs=n_jobs,
		@@ -529,19 +509,11 @@ class CsstMsDataManager:
		assert detector in self.available_detectors
		if post == ".fits":
		# no additional suffix
		if self.ver_sim in ["C5.2", "C6.1"]:
		fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
		f"{self.obs_id}_{detector:02d}{post}"
		else:
		fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
		f"{self.obs_id}_{detector:02d}_L1_V{self.version}{post}"
		else:
		if self.ver_sim in ["C5.2", "C6.1"]:
		fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
		f"{self.obs_id}_{detector:02d}_{post}"
		fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_"
		f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L1_V{self.version}{post}")
		else:
		fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
		f"{self.obs_id}_{detector:02d}_L1_V{self.version}_{post}"
		fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_"
		f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L1_V{self.version}_{post}")
		return os.path.join(self.dir_l1, fn)

		def get_refs(self, detector=6):
		@@ -730,7 +702,6 @@ class CsstMsDataManager:
		lines += f"- Target detectors = {self.target_detectors}\n"
		lines += f"- dir_l0 = {self.dir_l0}\n"
		lines += f"- dir_l1 = {self.dir_l1}\n"
		lines += f"- use_dfs = {self.use_dfs}\n"
		lines += f"- dfs_node = {self.dfs_node}\n"
		lines += f"- CSST_DFS_GATEWAY = " + os.getenv("CSST_DFS_GATEWAY") + "\n"
		return lines
		@@ -752,21 +723,6 @@ class CsstMsDataManager:
		def dfs_node(self):
		return self._dfs_node

		@dfs_node.setter
		def dfs_node(self, dfs_node):
		# for DFS configuration, defaults to "local", could be "pml"
		assert dfs_node in DFS_CONF.keys()
		self._dfs_node = dfs_node
		for k, v in DFS_CONF[dfs_node].items():
		os.environ[k] = v

		@staticmethod
		def dfs_setup(dfs_node="pml"):
		""" Setup DFS configuration. """
		assert dfs_node in DFS_CONF.keys()
		for k, v in DFS_CONF[dfs_node].items():
		os.environ[k] = v

		@staticmethod
		def dfs_is_available():
		""" Test if DFS works. """
		@@ -938,7 +894,6 @@ class CsstMsDataManager:
		dir_l1="/L1Pipeline/L1",
		path_aux="/L1Pipeline/aux",
		use_dfs=True,
		dfs_node="pml",
		clear_l1=False,
		dfs_root="/share/dfs",
		n_jobs=18,
		@@ -964,7 +919,7 @@ class CsstMsDataManager:
		os.chdir(dir_l1)

		print(f"Query obs_id={obs_id} ...", end="")
		records = CsstMsDataManager(dfs_node=dfs_node, verbose=False).dfs_L0DataApi.find(obs_id=obs_id)
		records = CsstMsDataManager(verbose=False).dfs_L0DataApi.find(obs_id=obs_id)
		print(f"{records['totalCount']} records obtained!")
		tbl = Table([_.__dict__ for _ in records["data"]])
		tbl.sort(["detector_no", "obs_type"])
		@@ -984,7 +939,6 @@ class CsstMsDataManager:
		dir_l1=dir_l1,
		path_aux=path_aux,
		use_dfs=use_dfs,
		dfs_node=dfs_node,
		n_jobs=n_jobs,
		backend=backend,
		device=device,
		@@ -1035,10 +989,6 @@ class CsstMsDataManager:
		pass


		# temporarily compatible with old interface
		CsstMbiDataManager = CsstMsDataManager


		class CsstMsFile(dict):
		def __init__(self, filepath, ver_sim="C6.2", dir_out=".", **kwargs):
		super(CsstMsFile, self).__init__()
		@@ -1090,11 +1040,11 @@ class CsstMsFile(dict):

		def fno(self, post=".fits"):
		if post.startswith("."):
		fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
		f"{self.obs_id}_{self.detector}_L1_V{self.version}{post}"
		fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_"
		f"{self.exp_stop}_{self.obs_id}_{self.detector}_L1_V{self.version}{post}")
		else:
		fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
		f"{self.obs_id}_{self.detector}_L1_V{self.version}_{post}"
		fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_"
		f"{self.exp_stop}_{self.obs_id}_{self.detector}_L1_V{self.version}_{post}")
		return fn

		def fpi(self):