Skip to content
GitLab
Commit 475ea658 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

rewrite CCDS class

parent b3a44a16
Pipeline #11699 passed with stage
Hide whitespace changes
Inline Side-by-side
csst_common/ccds/ccds.py
View file @ 475ea658
......@@ -10,134 +10,112 @@ Modified-History:
"""
import os
import logging
from typing import Optional
from ccds import client
class CCDS:
def __init__(
self,
use_oss=False,
ccds_root: str = "/ccds_root",
ccds_cache: str = "/pipeline/ccds_cache",
logger: Optional[logging.Logger] = None,
):
print("[CCDS] Setting CCDS root path ... ", end="")
self.use_oss = use_oss
self.ccds_root = ccds_root
print(self.ccds_root)
print("[CCDS] Setting CCDS cache path ... ", end="")
self.ccds_cache = ccds_cache
print(self.ccds_cache)
self.logger = logger
print("[CCDS] Setting CCDS environment variables ... ", end="")
os.environ["CCDS_MODE"] = "remote"
os.environ["CCDS_PATH"] = ccds_cache
os.environ["CCDS_OBSERVATORY"] = "csst"
print("Done")
print("[CCDS] Query for observatory ... ", end="")
self.observatory = client.get_default_observatory()
print(self.observatory)
assert (
self.observatory == "csst"
), f"observatory [{self.observatory}] is not `csst`!"
self.pmapname = self.operational_context = client.get_default_context(
self.observatory
)
print(f"[CCDS] Query for operational_context = {self.operational_context}... ")
self.is_available = False
def __repr__(self):
return (
f"[CCDS] url='{os.getenv('CCDS_SERVER_URL', default='')}' >\n"
# f" - is_available={self.is_available}\n"
f" - observatory='{self.observatory}'\n"
f" - operational_context='{self.operational_context}'\n"
f" - ccds_root='{self.ccds_root}'\n"
f" - ccdscache='{self.ccds_cache}'\n"
)
self.operational_context = client.get_default_context(self.observatory)
def get_refs(
self,
file_path: str = "./test.fits",
pmapname: str = "",
):
# Set pmapname
if pmapname:
print(f"[CCDS] Setting pmapname = {pmapname}... ")
self.pmapname = pmapname
try:
# dump mappings
client.dump_mappings(self.pmapname)
self.is_available = True
except BaseException as e:
print(f"[CCDS] Failed to get pmapname = {self.pmapname}:", e.args)
self.is_available = False
# print(f"[CCDS] operational_context = {self.operational_context}")
print(f"[CCDS] pmapname = {self.pmapname}")
# print("get min header")
hdrs = client.api.get_minimum_header(
self.pmapname, file_path, ignore_cache=False
file_path: str,
pmapname: Optional[str] = None,
use_oss=None,
) -> dict:
if not pmapname:
pmapname = self.operational_context
# dump pmap
client.dump_mappings(pmapname)
# get header
hdrs: dict = client.api.get_minimum_header(
pmapname, file_path, ignore_cache=False
)
# {'CAMERA': 'MS',
# 'CHIPID': '10',
# 'DATE-OBS': '2028-09-16T07:20:59',
# 'FILTER': 'GI',
# 'DATE-OBS': '2026-02-25T03:58:41.0',
# 'DETECTOR': '01',
# 'GAINLVL': '01',
# 'INSTRUME': 'MSC',
# 'REFTYPE': 'UNDEFINED'}
# print("get best ref")
refs = client.api.get_best_references(self.pmapname, hdrs)
# {'bias': 'csst_msc_ms_bias_10_000001.fits',
# 'dark': 'csst_msc_ms_dark_10_000001.fits',
# 'ledflat': 'csst_msc_ms_ledflat_10_000001.fits',
# 'shutter': "NOT FOUND parameter='CHIPID' value='10' is not in ['06', '07', '08', '09', '11', '12', '13',
# '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']"}
# assert ref_type in refs.keys(), f"ref_type [{ref_type}] not in {refs.keys()}"
# print("get file info")
refs_fp = dict()
for ref_type in refs.keys():
d = client.api.get_file_info(self.pmapname, refs[ref_type])
if d:
fp = os.path.join(self.ccds_root, d["file_path"])
assert os.path.exists(fp), f"File path [{fp}] does not exist!"
refs_fp[ref_type] = fp
else:
print(f"Failed to get file info for [{ref_type}:{refs[ref_type]}]")
# {'bias': '/ccds_root/references/msc/csst_msc_ms_bias_10_000001.fits',
# 'dark': '/ccds_root/references/msc/csst_msc_ms_dark_10_000001.fits',
# 'ledflat': '/ccds_root/references/msc/csst_msc_ms_ledflat_10_000001.fits'}
return refs_fp
# 'LEDTEMP': '173.0',
# 'REFTYPE': 'UNDEFINED',
# 'ROSPEED': '10.0'}
refs: dict = client.api.get_best_references(pmapname, hdrs)
# {'bias': 'csst_msc_ms_bias_01_000002.fits',
# 'countmbi': 'csst_msc_ms_countmbi_00_000001.pickle',
# 'countsls': 'csst_msc_ms_countsls_00_000001.pickle',
# 'cti': 'csst_msc_ms_cti_01_000001.fits',
# 'dark': 'csst_msc_ms_dark_01_000001.fits',
# 'deepcr': 'csst_msc_ms_deepcr_01_000001.pickle',
# 'detection': 'csst_msc_ms_detection_00_000001.pickle',
# 'element': 'csst_msc_ms_element_00_000005.fits',
# 'extract1d': 'csst_msc_ms_extract1d_01_000002.toml',
# 'flat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# 'gain': 'csst_msc_ms_gain_01_000003.fits',
# 'ledflat': 'csst_msc_ms_ledflat_01_000002.fits',
# 'ledflux': 'csst_msc_ms_ledflux_00_000005.fits',
# 'ooccfg': 'csst_msc_ms_ooccfg_00_000001.toml',
# 'pflat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# 'photflat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# 'psf': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# 'sensitivity': 'csst_msc_ms_sensitivity_01_000002.fits',
# 'shutter': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# 'skyflat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# 'zeropoint': 'csst_msc_ms_zeropoint_00_000010.fits'}
if self.logger:
self.logger.info(f"refs: {refs}")
else:
print(f"refs: {refs}")
refs_abspath = {}
if not use_oss:
# kick process NOT FOUND cases
for ref_type, ref_name in refs.items():
ref_info = client.api.get_file_info(pmapname, ref_name)
if ref_info:
ref_abspath = os.path.join(self.ccds_root, ref_info["file_path"])
assert os.path.exists(ref_abspath), ref_abspath
refs_abspath[ref_type] = ref_abspath
else:
# kick process NOT FOUND cases
for ref_type, ref_name in refs.items():
ref_info = client.api.get_file_info(pmapname, ref_name)
if ref_info:
ref_abspath = ref_info["file_path"]
# assert os.path.exists(ref_abspath), ref_abspath
refs_abspath[ref_type] = ref_abspath
if self.logger:
self.logger.info(f"refs_abspath: {refs_abspath}")
else:
print(f"refs_abspath: {refs_abspath}")
"""
file_path="/dfsroot/L0/MSC/SCIE/62030/10160000105/MS/CSST_MSC_MS_SCIE_20280916072059_20280916072329_10160000105_10_L0_V01.fits"
hdrs = client.api.get_minimum_header(c.operational_context, file_path, ignore_cache=True)
refs = client.api.get_best_references(c.operational_context, hdrs)
refs = crds.getrecommendations(hdrs, reftypes=None, context=c.operational_context, ignore_cache=True, observatory=c.observatory, fast=False)
refs = crds.getreferences(hdrs, reftypes=['bias'], context=c.operational_context, ignore_cache=False, observatory=c.observatory)
docker run --rm -it \
-v /share/crdsdata/data:/ccds_root \
-e CRDS_SERVER_URL=http://172.24.27.2:29000 \
-v /share/dfs:/dfsroot \
csst/csst-msc-l1-mbi bash
"""
if __name__ == "__main__":
c = CCDS()
refs = c.get_refs()
print(refs)
refs = c.retry(
c.get_refs,
3,
file_path="/dfsroot/L0/MSC/SCIE/62030/10160000105/MS/CSST_MSC_MS_SCIE_20280916072059_20280916072329_10160000105_11_L0_V01.fits",
)
print(refs)
# {'bias': '/ccds_root/references/msc/csst_msc_ms_bias_11_000001.fits',
# 'dark': '/ccds_root/references/msc/csst_msc_ms_dark_11_000001.fits',
# 'ledflat': '/ccds_root/references/msc/csst_msc_ms_ledflat_11_000001.fits',
# 'shutter': '/ccds_root/references/msc/csst_msc_ms_shutter_11_000001.fits'}
ccds = CCDS()
file_path = "/dfs_root/CSST_L0/test-msc-c9-25sqdeg-v3/MSC/20260225/MS/CSST_MSC_MS_WIDE_20260225035843_20260225040113_10100285453_01_L0_V01.fits"
pmapname = "csst_000094.pmap"
ccds.get_refs(file_path, pmapname=pmapname, use_oss=False)
csst_common/pipeline.py
View file @ 475ea658
......@@ -111,7 +111,12 @@ class Pipeline:
# DFS1, DFS2 & CCDS
self.dfs1 = csst_dfs_client
self.dfs2 = csst_fs
self.ccds = CCDS(ccds_root=self.ccds_root, ccds_cache=self.ccds_cache)
self.ccds = CCDS(
use_oss=self.use_oss,
ccds_root=self.ccds_root,
ccds_cache=self.ccds_cache,
logger=self.logger,
)
# exit code -> p.exit(reason="ccds_error")
self.EXIT_CODES = EXIT_CODES
......@@ -200,7 +205,7 @@ class Pipeline:
"""Initialize File object."""
return File(file_path, new_dir=self.dir_output)
def new(self, file_name="test.fits"):
def new(self, file_name="test.fits") -> str:
"""Create new file in output directory."""
return os.path.join(self.dir_output, file_name)
......@@ -259,7 +264,8 @@ class Pipeline:
return local_refs
# time operations
def now(self) -> str:
@staticmethod
def now() -> str:
"""Return ISOT format datetime using `astropy`."""
return Time.now().isot
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment