Skip to content
GitLab
Commit 475ea658 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

rewrite CCDS class

parent b3a44a16
Pipeline #11699 passed with stage
Hide whitespace changes
Inline Side-by-side
csst_common/ccds/ccds.py
View file @ 475ea658
...@@ -10,134 +10,112 @@ Modified-History: ...@@ -10,134 +10,112 @@ Modified-History:
""" """
import os import os
import logging
from typing import Optional
from ccds import client from ccds import client
class CCDS: class CCDS:
def __init__( def __init__(
self, self,
use_oss=False,
ccds_root: str = "/ccds_root", ccds_root: str = "/ccds_root",
ccds_cache: str = "/pipeline/ccds_cache", ccds_cache: str = "/pipeline/ccds_cache",
logger: Optional[logging.Logger] = None,
): ):
print("[CCDS] Setting CCDS root path ... ", end="") self.use_oss = use_oss
self.ccds_root = ccds_root self.ccds_root = ccds_root
print(self.ccds_root)
print("[CCDS] Setting CCDS cache path ... ", end="")
self.ccds_cache = ccds_cache self.ccds_cache = ccds_cache
print(self.ccds_cache) self.logger = logger
print("[CCDS] Setting CCDS environment variables ... ", end="")
os.environ["CCDS_MODE"] = "remote" os.environ["CCDS_MODE"] = "remote"
os.environ["CCDS_PATH"] = ccds_cache os.environ["CCDS_PATH"] = ccds_cache
os.environ["CCDS_OBSERVATORY"] = "csst" os.environ["CCDS_OBSERVATORY"] = "csst"
print("Done")
print("[CCDS] Query for observatory ... ", end="")
self.observatory = client.get_default_observatory() self.observatory = client.get_default_observatory()
print(self.observatory) self.operational_context = client.get_default_context(self.observatory)
assert (
self.observatory == "csst"
), f"observatory [{self.observatory}] is not `csst`!"
self.pmapname = self.operational_context = client.get_default_context(
self.observatory
)
print(f"[CCDS] Query for operational_context = {self.operational_context}... ")
self.is_available = False
def __repr__(self):
return (
f"[CCDS] url='{os.getenv('CCDS_SERVER_URL', default='')}' >\n"
# f" - is_available={self.is_available}\n"
f" - observatory='{self.observatory}'\n"
f" - operational_context='{self.operational_context}'\n"
f" - ccds_root='{self.ccds_root}'\n"
f" - ccdscache='{self.ccds_cache}'\n"
)
def get_refs( def get_refs(
self, self,
file_path: str = "./test.fits", file_path: str,
pmapname: str = "", pmapname: Optional[str] = None,
): use_oss=None,
# Set pmapname ) -> dict:
if pmapname:
print(f"[CCDS] Setting pmapname = {pmapname}... ") if not pmapname:
self.pmapname = pmapname pmapname = self.operational_context
try:
# dump mappings # dump pmap
client.dump_mappings(self.pmapname) client.dump_mappings(pmapname)
self.is_available = True
except BaseException as e: # get header
print(f"[CCDS] Failed to get pmapname = {self.pmapname}:", e.args) hdrs: dict = client.api.get_minimum_header(
self.is_available = False pmapname, file_path, ignore_cache=False
# print(f"[CCDS] operational_context = {self.operational_context}")
print(f"[CCDS] pmapname = {self.pmapname}")
# print("get min header")
hdrs = client.api.get_minimum_header(
self.pmapname, file_path, ignore_cache=False
) )
# {'CAMERA': 'MS', # {'CAMERA': 'MS',
# 'CHIPID': '10', # 'DATE-OBS': '2026-02-25T03:58:41.0',
# 'DATE-OBS': '2028-09-16T07:20:59', # 'DETECTOR': '01',
# 'FILTER': 'GI', # 'GAINLVL': '01',
# 'INSTRUME': 'MSC', # 'INSTRUME': 'MSC',
# 'REFTYPE': 'UNDEFINED'} # 'LEDTEMP': '173.0',
# print("get best ref") # 'REFTYPE': 'UNDEFINED',
refs = client.api.get_best_references(self.pmapname, hdrs) # 'ROSPEED': '10.0'}
# {'bias': 'csst_msc_ms_bias_10_000001.fits', refs: dict = client.api.get_best_references(pmapname, hdrs)
# 'dark': 'csst_msc_ms_dark_10_000001.fits', # {'bias': 'csst_msc_ms_bias_01_000002.fits',
# 'ledflat': 'csst_msc_ms_ledflat_10_000001.fits', # 'countmbi': 'csst_msc_ms_countmbi_00_000001.pickle',
# 'shutter': "NOT FOUND parameter='CHIPID' value='10' is not in ['06', '07', '08', '09', '11', '12', '13', # 'countsls': 'csst_msc_ms_countsls_00_000001.pickle',
# '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']"} # 'cti': 'csst_msc_ms_cti_01_000001.fits',
# assert ref_type in refs.keys(), f"ref_type [{ref_type}] not in {refs.keys()}" # 'dark': 'csst_msc_ms_dark_01_000001.fits',
# print("get file info") # 'deepcr': 'csst_msc_ms_deepcr_01_000001.pickle',
refs_fp = dict() # 'detection': 'csst_msc_ms_detection_00_000001.pickle',
for ref_type in refs.keys(): # 'element': 'csst_msc_ms_element_00_000005.fits',
d = client.api.get_file_info(self.pmapname, refs[ref_type]) # 'extract1d': 'csst_msc_ms_extract1d_01_000002.toml',
if d: # 'flat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
fp = os.path.join(self.ccds_root, d["file_path"]) # 'gain': 'csst_msc_ms_gain_01_000003.fits',
assert os.path.exists(fp), f"File path [{fp}] does not exist!" # 'ledflat': 'csst_msc_ms_ledflat_01_000002.fits',
refs_fp[ref_type] = fp # 'ledflux': 'csst_msc_ms_ledflux_00_000005.fits',
else: # 'ooccfg': 'csst_msc_ms_ooccfg_00_000001.toml',
print(f"Failed to get file info for [{ref_type}:{refs[ref_type]}]") # 'pflat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# {'bias': '/ccds_root/references/msc/csst_msc_ms_bias_10_000001.fits', # 'photflat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# 'dark': '/ccds_root/references/msc/csst_msc_ms_dark_10_000001.fits', # 'psf': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# 'ledflat': '/ccds_root/references/msc/csst_msc_ms_ledflat_10_000001.fits'} # 'sensitivity': 'csst_msc_ms_sensitivity_01_000002.fits',
return refs_fp # 'shutter': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# 'skyflat': "NOT FOUND parameter='DETECTOR' value='01' is not in ['06', '07', '08', '09', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']",
# 'zeropoint': 'csst_msc_ms_zeropoint_00_000010.fits'}
if self.logger:
self.logger.info(f"refs: {refs}")
else:
print(f"refs: {refs}")
refs_abspath = {}
if not use_oss:
# kick process NOT FOUND cases
for ref_type, ref_name in refs.items():
ref_info = client.api.get_file_info(pmapname, ref_name)
if ref_info:
ref_abspath = os.path.join(self.ccds_root, ref_info["file_path"])
assert os.path.exists(ref_abspath), ref_abspath
refs_abspath[ref_type] = ref_abspath
else:
# kick process NOT FOUND cases
for ref_type, ref_name in refs.items():
ref_info = client.api.get_file_info(pmapname, ref_name)
if ref_info:
ref_abspath = ref_info["file_path"]
# assert os.path.exists(ref_abspath), ref_abspath
refs_abspath[ref_type] = ref_abspath
if self.logger:
self.logger.info(f"refs_abspath: {refs_abspath}")
else:
print(f"refs_abspath: {refs_abspath}")
"""
file_path="/dfsroot/L0/MSC/SCIE/62030/10160000105/MS/CSST_MSC_MS_SCIE_20280916072059_20280916072329_10160000105_10_L0_V01.fits"
hdrs = client.api.get_minimum_header(c.operational_context, file_path, ignore_cache=True)
refs = client.api.get_best_references(c.operational_context, hdrs)
refs = crds.getrecommendations(hdrs, reftypes=None, context=c.operational_context, ignore_cache=True, observatory=c.observatory, fast=False)
refs = crds.getreferences(hdrs, reftypes=['bias'], context=c.operational_context, ignore_cache=False, observatory=c.observatory)
docker run --rm -it \
-v /share/crdsdata/data:/ccds_root \
-e CRDS_SERVER_URL=http://172.24.27.2:29000 \
-v /share/dfs:/dfsroot \
csst/csst-msc-l1-mbi bash
"""
if __name__ == "__main__": if __name__ == "__main__":
c = CCDS() ccds = CCDS()
refs = c.get_refs() file_path = "/dfs_root/CSST_L0/test-msc-c9-25sqdeg-v3/MSC/20260225/MS/CSST_MSC_MS_WIDE_20260225035843_20260225040113_10100285453_01_L0_V01.fits"
print(refs) pmapname = "csst_000094.pmap"
ccds.get_refs(file_path, pmapname=pmapname, use_oss=False)
refs = c.retry(
c.get_refs,
3,
file_path="/dfsroot/L0/MSC/SCIE/62030/10160000105/MS/CSST_MSC_MS_SCIE_20280916072059_20280916072329_10160000105_11_L0_V01.fits",
)
print(refs)
# {'bias': '/ccds_root/references/msc/csst_msc_ms_bias_11_000001.fits',
# 'dark': '/ccds_root/references/msc/csst_msc_ms_dark_11_000001.fits',
# 'ledflat': '/ccds_root/references/msc/csst_msc_ms_ledflat_11_000001.fits',
# 'shutter': '/ccds_root/references/msc/csst_msc_ms_shutter_11_000001.fits'}
csst_common/pipeline.py
View file @ 475ea658
...@@ -111,7 +111,12 @@ class Pipeline: ...@@ -111,7 +111,12 @@ class Pipeline:
# DFS1, DFS2 & CCDS # DFS1, DFS2 & CCDS
self.dfs1 = csst_dfs_client self.dfs1 = csst_dfs_client
self.dfs2 = csst_fs self.dfs2 = csst_fs
self.ccds = CCDS(ccds_root=self.ccds_root, ccds_cache=self.ccds_cache) self.ccds = CCDS(
use_oss=self.use_oss,
ccds_root=self.ccds_root,
ccds_cache=self.ccds_cache,
logger=self.logger,
)
# exit code -> p.exit(reason="ccds_error") # exit code -> p.exit(reason="ccds_error")
self.EXIT_CODES = EXIT_CODES self.EXIT_CODES = EXIT_CODES
...@@ -200,7 +205,7 @@ class Pipeline: ...@@ -200,7 +205,7 @@ class Pipeline:
"""Initialize File object.""" """Initialize File object."""
return File(file_path, new_dir=self.dir_output) return File(file_path, new_dir=self.dir_output)
def new(self, file_name="test.fits"): def new(self, file_name="test.fits") -> str:
"""Create new file in output directory.""" """Create new file in output directory."""
return os.path.join(self.dir_output, file_name) return os.path.join(self.dir_output, file_name)
...@@ -259,7 +264,8 @@ class Pipeline: ...@@ -259,7 +264,8 @@ class Pipeline:
return local_refs return local_refs
# time operations # time operations
def now(self) -> str: @staticmethod
def now() -> str:
"""Return ISOT format datetime using `astropy`.""" """Return ISOT format datetime using `astropy`."""
return Time.now().isot return Time.now().isot
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment