rewrite CCDS class (475ea658) · Commits · Wu Maochun / csst_common

csst_common/ccds/ccds.py

+82 −104

Original line number	Original line	Diff line number	Diff line
	@@ -10,134 +10,112 @@ Modified-History:
	"""		"""

	import os		import os
			import logging
			from typing import Optional

	from ccds import client		from ccds import client


	class CCDS:		class CCDS:

	def __init__(		def __init__(
	self,		self,
			use_oss=False,
	ccds_root: str = "/ccds_root",		ccds_root: str = "/ccds_root",
	ccds_cache: str = "/pipeline/ccds_cache",		ccds_cache: str = "/pipeline/ccds_cache",
			logger: Optional[logging.Logger] = None,
	):		):
	print("[CCDS] Setting CCDS root path ... ", end="")		self.use_oss = use_oss
	self.ccds_root = ccds_root		self.ccds_root = ccds_root
	print(self.ccds_root)

	print("[CCDS] Setting CCDS cache path ... ", end="")
	self.ccds_cache = ccds_cache		self.ccds_cache = ccds_cache
	print(self.ccds_cache)		self.logger = logger

	print("[CCDS] Setting CCDS environment variables ... ", end="")
	os.environ["CCDS_MODE"] = "remote"		os.environ["CCDS_MODE"] = "remote"
	os.environ["CCDS_PATH"] = ccds_cache		os.environ["CCDS_PATH"] = ccds_cache
	os.environ["CCDS_OBSERVATORY"] = "csst"		os.environ["CCDS_OBSERVATORY"] = "csst"
	print("Done")

	print("[CCDS] Query for observatory ... ", end="")
	self.observatory = client.get_default_observatory()		self.observatory = client.get_default_observatory()
	print(self.observatory)		self.operational_context = client.get_default_context(self.observatory)
	assert (
	self.observatory == "csst"
	), f"observatory [{self.observatory}] is not `csst`!"

	self.pmapname = self.operational_context = client.get_default_context(
	self.observatory
	)
	print(f"[CCDS] Query for operational_context = {self.operational_context}... ")

	self.is_available = False

	def __repr__(self):
	return (
	f"[CCDS] url='{os.getenv('CCDS_SERVER_URL', default='')}' >\n"
	# f" - is_available={self.is_available}\n"
	f" - observatory='{self.observatory}'\n"
	f" - operational_context='{self.operational_context}'\n"
	f" - ccds_root='{self.ccds_root}'\n"
	f" - ccdscache='{self.ccds_cache}'\n"
	)

	def get_refs(		def get_refs(
	self,		self,
	file_path: str = "./test.fits",		file_path: str,
	pmapname: str = "",		pmapname: Optional[str] = None,
	):		use_oss=None,
	# Set pmapname		) -> dict:
	if pmapname:
	print(f"[CCDS] Setting pmapname = {pmapname}... ")		if not pmapname:
	self.pmapname = pmapname		pmapname = self.operational_context
	try:
	# dump mappings		# dump pmap
	client.dump_mappings(self.pmapname)		client.dump_mappings(pmapname)
	self.is_available = True
	except BaseException as e:		# get header
	print(f"[CCDS] Failed to get pmapname = {self.pmapname}:", e.args)		hdrs: dict = client.api.get_minimum_header(
	self.is_available = False		pmapname, file_path, ignore_cache=False

	# print(f"[CCDS] operational_context = {self.operational_context}")
	print(f"[CCDS] pmapname = {self.pmapname}")
	# print("get min header")
	hdrs = client.api.get_minimum_header(
	self.pmapname, file_path, ignore_cache=False
	)		)
	# {'CAMERA': 'MS',		# {'CAMERA': 'MS',
	# 'CHIPID': '10',		# 'DATE-OBS': '2026-02-25T03:58:41.0',
	# 'DATE-OBS': '2028-09-16T07:20:59',		# 'DETECTOR': '01',
	# 'FILTER': 'GI',		# 'GAINLVL': '01',
	# 'INSTRUME': 'MSC',		# 'INSTRUME': 'MSC',
	# 'REFTYPE': 'UNDEFINED'}		# 'LEDTEMP': '173.0',
	# print("get best ref")		# 'REFTYPE': 'UNDEFINED',
	refs = client.api.get_best_references(self.pmapname, hdrs)		# 'ROSPEED': '10.0'}
	# {'bias': 'csst_msc_ms_bias_10_000001.fits',		refs: dict = client.api.get_best_references(pmapname, hdrs)
	# 'dark': 'csst_msc_ms_dark_10_000001.fits',		# {'bias': 'csst_msc_ms_bias_01_000002.fits',
	# 'ledflat': 'csst_msc_ms_ledflat_10_000001.fits',		# 'countmbi': 'csst_msc_ms_countmbi_00_000001.pickle',
	# 'shutter': "NOT FOUND parameter='CHIPID' value='10' is not in ['06', '07', '08', '09', '11', '12', '13',		# 'countsls': 'csst_msc_ms_countsls_00_000001.pickle',
	# '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']"}		# 'cti': 'csst_msc_ms_cti_01_000001.fits',
	# assert ref_type in refs.keys(), f"ref_type [{ref_type}] not in {refs.keys()}"		# 'dark': 'csst_msc_ms_dark_01_000001.fits',
	# print("get file info")		# 'deepcr': 'csst_msc_ms_deepcr_01_000001.pickle',
	refs_fp = dict()		# 'detection': 'csst_msc_ms_detection_00_000001.pickle',
	for ref_type in refs.keys():		# 'element': 'csst_msc_ms_element_00_000005.fits',
	d = client.api.get_file_info(self.pmapname, refs[ref_type])		# 'extract1d': 'csst_msc_ms_extract1d_01_000002.toml',
	if d:		# 'flat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
	fp = os.path.join(self.ccds_root, d["file_path"])		# 'gain': 'csst_msc_ms_gain_01_000003.fits',
	assert os.path.exists(fp), f"File path [{fp}] does not exist!"		# 'ledflat': 'csst_msc_ms_ledflat_01_000002.fits',
	refs_fp[ref_type] = fp		# 'ledflux': 'csst_msc_ms_ledflux_00_000005.fits',
			# 'ooccfg': 'csst_msc_ms_ooccfg_00_000001.toml',
			# 'pflat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
			# 'photflat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
			# 'psf': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
			# 'sensitivity': 'csst_msc_ms_sensitivity_01_000002.fits',
			# 'shutter': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
			# 'skyflat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
			# 'zeropoint': 'csst_msc_ms_zeropoint_00_000010.fits'}
			if self.logger:
			self.logger.info(f"refs: {refs}")
			else:
			print(f"refs: {refs}")

			refs_abspath = {}

			if not use_oss:
			# kick process NOT FOUND cases
			for ref_type, ref_name in refs.items():
			ref_info = client.api.get_file_info(pmapname, ref_name)
			if ref_info:
			ref_abspath = os.path.join(self.ccds_root, ref_info["file_path"])
			assert os.path.exists(ref_abspath), ref_abspath
			refs_abspath[ref_type] = ref_abspath
	else:		else:
	print(f"Failed to get file info for [{ref_type}:{refs[ref_type]}]")		# kick process NOT FOUND cases
	# {'bias': '/ccds_root/references/msc/csst_msc_ms_bias_10_000001.fits',		for ref_type, ref_name in refs.items():
	# 'dark': '/ccds_root/references/msc/csst_msc_ms_dark_10_000001.fits',		ref_info = client.api.get_file_info(pmapname, ref_name)
	# 'ledflat': '/ccds_root/references/msc/csst_msc_ms_ledflat_10_000001.fits'}		if ref_info:
	return refs_fp		ref_abspath = ref_info["file_path"]
			# assert os.path.exists(ref_abspath), ref_abspath
			refs_abspath[ref_type] = ref_abspath

			if self.logger:
			self.logger.info(f"refs_abspath: {refs_abspath}")
			else:
			print(f"refs_abspath: {refs_abspath}")


	"""
	file_path="/dfsroot/L0/MSC/SCIE/62030/10160000105/MS/CSST_MSC_MS_SCIE_20280916072059_20280916072329_10160000105_10_L0_V01.fits"
	hdrs = client.api.get_minimum_header(c.operational_context, file_path, ignore_cache=True)
	refs = client.api.get_best_references(c.operational_context, hdrs)
	refs = crds.getrecommendations(hdrs, reftypes=None, context=c.operational_context, ignore_cache=True, observatory=c.observatory, fast=False)
	refs = crds.getreferences(hdrs, reftypes=['bias'], context=c.operational_context, ignore_cache=False, observatory=c.observatory)


	docker run --rm -it \
	-v /share/crdsdata/data:/ccds_root \
	-e CRDS_SERVER_URL=http://172.24.27.2:29000 \
	-v /share/dfs:/dfsroot \
	csst/csst-msc-l1-mbi bash
	"""

	if __name__ == "__main__":		if __name__ == "__main__":
	c = CCDS()		ccds = CCDS()
	refs = c.get_refs()		file_path = "/dfs_root/CSST_L0/test-msc-c9-25sqdeg-v3/MSC/20260225/MS/CSST_MSC_MS_WIDE_20260225035843_20260225040113_10100285453_01_L0_V01.fits"
	print(refs)		pmapname = "csst_000094.pmap"
			ccds.get_refs(file_path, pmapname=pmapname, use_oss=False)
	refs = c.retry(
	c.get_refs,
	3,
	file_path="/dfsroot/L0/MSC/SCIE/62030/10160000105/MS/CSST_MSC_MS_SCIE_20280916072059_20280916072329_10160000105_11_L0_V01.fits",
	)
	print(refs)
	# {'bias': '/ccds_root/references/msc/csst_msc_ms_bias_11_000001.fits',
	# 'dark': '/ccds_root/references/msc/csst_msc_ms_dark_11_000001.fits',
	# 'ledflat': '/ccds_root/references/msc/csst_msc_ms_ledflat_11_000001.fits',
	# 'shutter': '/ccds_root/references/msc/csst_msc_ms_shutter_11_000001.fits'}

csst_common/pipeline.py

+9 −3

Original line number	Original line	Diff line number	Diff line
	@@ -111,7 +111,12 @@ class Pipeline:
	# DFS1, DFS2 & CCDS		# DFS1, DFS2 & CCDS
	self.dfs1 = csst_dfs_client		self.dfs1 = csst_dfs_client
	self.dfs2 = csst_fs		self.dfs2 = csst_fs
	self.ccds = CCDS(ccds_root=self.ccds_root, ccds_cache=self.ccds_cache)		self.ccds = CCDS(
			use_oss=self.use_oss,
			ccds_root=self.ccds_root,
			ccds_cache=self.ccds_cache,
			logger=self.logger,
			)

	# exit code -> p.exit(reason="ccds_error")		# exit code -> p.exit(reason="ccds_error")
	self.EXIT_CODES = EXIT_CODES		self.EXIT_CODES = EXIT_CODES
	@@ -200,7 +205,7 @@ class Pipeline:
	"""Initialize File object."""		"""Initialize File object."""
	return File(file_path, new_dir=self.dir_output)		return File(file_path, new_dir=self.dir_output)

	def new(self, file_name="test.fits"):		def new(self, file_name="test.fits") -> str:
	"""Create new file in output directory."""		"""Create new file in output directory."""
	return os.path.join(self.dir_output, file_name)		return os.path.join(self.dir_output, file_name)

	@@ -259,7 +264,8 @@ class Pipeline:
	return local_refs		return local_refs

	# time operations		# time operations
	def now(self) -> str:		@staticmethod
			def now() -> str:
	"""Return ISOT format datetime using `astropy`."""		"""Return ISOT format datetime using `astropy`."""
	return Time.now().isot		return Time.now().isot