compatible with new L0 data model (c283cd15) · Commits · csst-pipeline / csst_common

csst_common/data/csst_params.yml

+1 −0

Original line number	Diff line number	Diff line
		@@ -77,6 +77,7 @@ sls: # slit-less spectra

		sim: # simulation
		versions:
		- C6.2
		- C6.1
		- C5.2

csst_common/data_manager.py

+81 −87

Original line number	Diff line number	Diff line
		@@ -13,7 +13,6 @@ Modified-History:
		2022-11-06, Bo Zhang, deleted CsstMbiDataManager
		2022-11-20, Bo Zhang, added DFS APIs
		"""
		import datetime
		import glob
		import os
		import re
		@@ -93,15 +92,15 @@ class CsstMsDataManager:
		The exposure start time in ``yyyymmddhhmmss`` format.
		exp_stop : int
		The exposure start time in ``yyyymmddhhmmss`` format.
		_telescope : str
		telescope : str
		The telescope name. Defaults to ``CSST`` for C5.2 simulation.
		_instrument : str
		instrument : str
		The instrument name. Defaults to ``MSC`` for C5.2 simulation.
		_survey : str
		project : str
		The survey name. Defaults to ``MS`` for C5.2 simulation.
		obs_type : str
		The image type signature for science images. Defualts to ``SCI`` for C5.2 simulation.
		l0_post : str
		level : str
		The postfix. Defaults to ``L0_1`` for C5.2 simulation.
		log_ppl : str
		The pipeline log file name.
		@@ -124,27 +123,26 @@ class CsstMsDataManager:

		Examples
		--------
		>>> dm_mbi = CsstMsDataManager(...)
		>>> dm = CsstMsDataManager.from_dir(
		>>> ver_sim="C6.2", datatype="mbi", dir_l0="/Users/cham/L1Pipeline/L0_MBI_C6.2", dir_l1=".")
		>>> # access L0 directory
		>>> dm_mbi.dir_l0
		>>> dm.dir_l0
		>>> # access L1 directory
		>>> dm_mbi.dir_l1
		>>> dm.dir_l1
		>>> # access path_aux
		>>> dm_mbi.path_aux
		>>> # access ver_sim
		>>> dm_mbi.ver_sim
		>>> dm.path_aux
		>>> # access target detectors
		>>> dm_mbi.target_detectors
		>>> dm.target_detectors
		>>> # access available detectors
		>>> dm_mbi.available_detectors
		>>> # define an L1 file (detector-specified)
		>>> dm_mbi.l1_detector(detector=6)
		>>> dm.available_detectors
		>>> # generate an L1 file path (detector-specified)
		>>> dm.l1_detector(detector=6, post="IMG.fits")
		>>> # define an L1 file (non-detector-specified)
		>>> dm_mbi.l1_file("flipped_image.fits")
		>>> dm.l1_file("flipped_image.fits")
		"""

		# TODO: update docstring
		def __init__(self,
		ver_sim: str = "C5.2",
		ver_sim: str = "C6.2",
		datatype: str = "mbi",
		available_detectors: Union[None, list] = None,
		target_detectors: Union[None, list, int] = None,
		@@ -154,14 +152,16 @@ class CsstMsDataManager:
		use_dfs: bool = False,
		dfs_node: str = "kmust",
		dfs_root: str = "/share/dfs",
		obs_id: str = "100000100",
		telescope: str = "CSST",
		instrument: str = "MSC",
		project: str = "MS",
		obs_type: str = "SCI",
		exp_start: int = "20270810081950",
		exp_stop: int = "20270810082220",
		_telescope: str = "CSST",
		_instrument: str = "MSC",
		_survey: str = "MS",
		obs_type: str = "SCI",
		l0_post: str = "L0_1",
		obs_id: str = "100000100",
		level: str = "0",
		version: str = "01",
		ext: str = "fits",
		log_ppl: str = "",
		log_mod: str = "",
		clear_dir: bool = False,
		@@ -214,11 +214,13 @@ class CsstMsDataManager:
		self.exp_stop = exp_stop

		# file name components
		self._telescope = _telescope
		self._instrument = _instrument
		self._survey = _survey
		self.telescope = telescope
		self.instrument = instrument
		self.project = project
		self.obs_type = obs_type
		self.l0_post = l0_post
		self.level = level
		self.version = version
		self.ext = ext

		# DFS configuration
		self.use_dfs = use_dfs
		@@ -353,11 +355,12 @@ class CsstMsDataManager:

		@staticmethod
		def from_dir(
		ver_sim="C5.2",
		ver_sim="C6.2",
		datatype="mbi",
		dir_l0=".",
		dir_l1=".",
		path_aux="",
		pattern="CSST_MSC__SCI_.fits",
		use_dfs=False,
		dfs_node="kmust",
		log_ppl="csst-l1ppl.log",
		@@ -372,23 +375,33 @@ class CsstMsDataManager:
		assert ver_sim in CP["sim"]["versions"]

		# glob files
		fps_img = CsstMsDataManager.glob_image(dir_l0, ver_sim=ver_sim)
		fps_img = CsstMsDataManager.glob_image(dir_l0=dir_l0, pattern=pattern)
		if len(fps_img) == 0:
		raise FileNotFoundError(f"No file found in dir_l0: {dir_l0}")
		raise FileNotFoundError(f"No file found with pattern {pattern} in {dir_l0}")

		# parse filename
		pattern = re.compile(
		r"(?P<telescope>[A-Z]+)_"
		r"(?P<instrument>[A-Z]+)_"
		r"(?P<project>[A-Z]+)_"
		r"(?P<obs_type>[A-Z]+)_"
		r"(?P<exp_start>[0-9]{14})_"
		r"(?P<exp_stop>[0-9]{14})_"
		r"(?P<obs_id>[0-9]{11})_"
		r"(?P<detector>[0-9]{2})_"
		r"L(?P<level>[0-9]{1})_"
		r"V(?P<version>[0-9]{2})"
		r".(?P<ext>[a-z]{4})"
		)
		mo = re.fullmatch(pattern, fps_img[0])
		assert mo is not None
		mogd = mo.groupdict()
		mogd.pop("detector")

		# available detectors
		available_detectors = [int(re.split(r"[_.]", fp)[7]) for fp in fps_img]
		available_detectors = [int(re.fullmatch(pattern, fp)["detector"]) for fp in fps_img]
		available_detectors.sort()

		# parse info
		(_telescope, _instrument, _survey, obs_type,
		exp_start, exp_stop, obs_id,
		_detector, *l0_post, _ext) = re.split(r"[_.]", fps_img[0])

		# exp_start = int(exp_start)
		# exp_stop = int(exp_stop)
		# obs_id = int(obs_id)

		return CsstMsDataManager(
		ver_sim=ver_sim,
		datatype=datatype,
		@@ -399,81 +412,63 @@ class CsstMsDataManager:
		path_aux=path_aux, # bias dark flat
		use_dfs=use_dfs,
		dfs_node=dfs_node,
		obs_id=obs_id,
		exp_start=exp_start,
		exp_stop=exp_stop,
		_telescope=_telescope,
		_instrument=_instrument,
		_survey=_survey,
		obs_type=obs_type,
		l0_post="_".join(l0_post),
		log_ppl=log_ppl,
		log_mod=log_mod,
		n_jobs=n_jobs,
		backend=backend,
		device=device,
		**mogd,
		**kwargs
		)

		@staticmethod
		def glob_image(dir_l0, ver_sim="C5.2"):
		def glob_image(dir_l0, pattern="CSST_MSC__SCI_.fits"):
		""" glob files in L0 data directory """
		assert ver_sim in CP["sim"]["versions"]
		pattern = os.path.join(dir_l0, "CSST_MSC_MS_SCI_*.fits")
		fps = glob.glob(pattern)
		fps = glob.glob(os.path.join(dir_l0, pattern))
		fps = [os.path.basename(fp) for fp in fps]
		fps.sort()

		print("{} files found with pattern: {}".format(len(fps), pattern))
		return fps

		@staticmethod
		def glob_cat(dir_l0, ver_sim="C5"):
		""" glob input catalogs in L0 data directory """
		assert ver_sim in CP["sim"]["versions"]
		pattern = os.path.join(dir_l0, "MSC_*.cat")
		fps = glob.glob(pattern)
		fps = [os.path.basename(fp) for fp in fps]
		fps.sort()

		print("@DM.glob_dir: {} files found with pattern: {}".format(len(fps), pattern))
		return fps

		def l0_id(self, detector=6):
		""" Level0 ID, consistent with DFS. """
		return f"{self.obs_id}{detector:02d}"

		def l0_cat(self, detector=6):
		""" the L0 cat file path"""
		assert self.ver_sim in CP["sim"]["versions"]
		fn = "{}_{}_chip_{:02d}_filt_{}.cat".format(
		self._instrument, self.obs_id, detector, self.detector2filter[detector])
		return os.path.join(self.dir_l0, fn)
		assert detector in self.available_detectors
		fn = f"{self.instrument}_{self.obs_id}_chip_{detector:02d}_filt_{self.detector2filter[detector]}.cat"
		fp = os.path.join(self.dir_l0, fn)
		assert os.path.exists(fp)
		return fp

		def l0_log(self, detector=6):
		""" L0 log file path """
		assert self.ver_sim in CP["sim"]["versions"]
		fn = "{}_{}_chip_{:02d}_filt_{}.log".format(
		self._instrument, self.obs_id, detector, self.detector2filter[detector])
		return os.path.join(self.dir_l0, fn)
		assert detector in self.available_detectors
		fn = f"{self.instrument}_{self.obs_id}_chip_{detector:02d}_filt_{self.detector2filter[detector]}.log"
		fp = os.path.join(self.dir_l0, fn)
		assert os.path.exists(fp)
		return fp

		def l0_detector(self, detector=6):
		""" L0 detector-specific image file path """
		assert self.ver_sim in CP["sim"]["versions"]
		fn = "{}_{}_{}_SCI_{}_{}_{}_{:02d}_L0_1.fits".format(
		self._telescope, self._instrument, self._survey,
		self.exp_start, self.exp_stop, self.obs_id, detector)
		assert detector in self.available_detectors
		fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
		f"{self.obs_id}_{detector:02d}_L{self.level}_V{self.version}.{self.ext}"
		fp = os.path.join(self.dir_l0, fn)
		assert os.path.exists(fp)
		return os.path.join(self.dir_l0, fn)

		def l0_crs(self, detector=6):
		""" L0 cosmic ray file path """
		assert self.ver_sim in CP["sim"]["versions"]
		fn = "{}_{}_{}_CRS_{}_{}_{}_{:02d}_L0_1.fits".format(
		self._telescope, self._instrument, self._survey,
		self.exp_start, self.exp_stop, self.obs_id, detector)
		assert detector in self.available_detectors
		fn = f"{self.telescope}_{self.instrument}_{self.project}_CRS_{self.exp_start}_{self.exp_stop}_" \
		f"{self.obs_id}_{detector:02d}_L{self.level}_V{self.version}.{self.ext}"
		fp = os.path.join(self.dir_l0, fn)
		assert os.path.exists(fp)
		return os.path.join(self.dir_l0, fn)

		def l1_detector(self, detector=6, post="img.fits"):
		def l1_detector(self, detector=6, post="IMG.fits"):
		""" generate L1 file path

		Parameters
		@@ -489,10 +484,9 @@ class CsstMsDataManager:
		L1 file path

		"""
		assert self.ver_sim in CP["sim"]["versions"]
		fn = "{}_{}_{}_SCI_{}_{}_{}_{:02d}_{}".format(
		self._telescope, self._instrument, self._survey,
		self.exp_start, self.exp_stop, self.obs_id, detector, post)
		assert detector in self.available_detectors
		fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \
		f"{self.obs_id}_{detector:02d}_L1_V{self.version}_{post}"
		return os.path.join(self.dir_l1, fn)

		def get_bias(self, detector=6):