From b026df9708690fff6ed47ba679fdd8d136083d86 Mon Sep 17 00:00:00 2001 From: BO ZHANG Date: Sun, 10 Dec 2023 17:35:48 +0800 Subject: [PATCH] major updates --- .gitignore | 1 + Makefile | 2 +- README.md | 39 --- csst_common/__init__.py | 10 +- csst_common/ccds/slsconf.py | 350 ++++++++++++++++++++ csst_common/coordinates.py | 18 - csst_common/crds/slsconf.py | 136 -------- csst_common/data/node_conf.yml | 18 - csst_common/data_manager.py | 490 ++++++++++++++++------------ csst_common/file_recorder.py | 131 -------- csst_common/io.py | 10 + csst_common/logger.py | 6 +- csst_common/params.py | 9 +- csst_common/pipeline.py | 194 ++++++----- csst_common/utils/__init__.py | 1 - csst_common/utils/_io.py | 15 - csst_common/utils/_module_docstr.py | 4 +- csst_common/utils/tempfile.py | 1 - csst_common/wrapper.py | 113 ------- tests/test_data_manager.py | 100 +++--- tests/test_file_recorder.py | 22 -- tests/test_params.py | 16 - 22 files changed, 818 insertions(+), 868 deletions(-) create mode 100644 csst_common/ccds/slsconf.py delete mode 100644 csst_common/coordinates.py delete mode 100644 csst_common/crds/slsconf.py delete mode 100644 csst_common/data/node_conf.yml delete mode 100644 csst_common/file_recorder.py create mode 100644 csst_common/io.py delete mode 100644 csst_common/utils/_io.py delete mode 100644 csst_common/wrapper.py delete mode 100644 tests/test_file_recorder.py delete mode 100644 tests/test_params.py diff --git a/.gitignore b/.gitignore index 4cfa1a9..984823c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +unit_test_data_root .idea/* *.png *.DS_Store* diff --git a/Makefile b/Makefile index 535cd03..7e25110 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,6 @@ uninstall: pip uninstall $(PKG) -y test: - coverage run -m pytest . --import-mode=importlib --cov-report=html --cov-report=term-missing + coverage run -m pytest . --cov=$(PKG) --import-mode=importlib --cov-report=html --cov-report=term-missing coverage report -m rm -rf .coverage .pytest_cache diff --git a/README.md b/README.md index 5b8a294..be09326 100644 --- a/README.md +++ b/README.md @@ -44,42 +44,3 @@ from csst_common.params import CSST_PARAMS as CP print(CP) print(CP["mbi"]["detector2filter"]) ``` - -how to use data_manager -```python -from csst_common.data_manager import CsstMbiDataManager -dm = CsstMbiDataManager(...) -# access L0 directory -dm.dir_l0 -# access L1 directory -dm.dir_l1 -# access dir_pcref -dm.dir_pcref -# access path_aux -dm.path_aux -# access ver_sim -dm.ver_sim -# access target detectors -dm.target_detectors -# access available detectors -dm.available_detectors -# define an L1 file (detector-specified) -dm.l1_detector(detector=6) -# define an L1 file (non-detector-specified) -dm.l1_file("flipped_image.fits") -``` - -a shortcut for test dataset - -```python -from csst_common.data_manager import CsstMbiDataManager - -CsstMbiDataManager.quickstart(ver_sim="C5.2", dir_l1=".", exposure_id=100) -``` - -## algorithm description - -The `csst_common` provides some common modules for CSST pipeline. - -- `csst_common.parameters` -- `csst_common.data_manager` diff --git a/csst_common/__init__.py b/csst_common/__init__.py index 86ff9b2..f99ff0c 100644 --- a/csst_common/__init__.py +++ b/csst_common/__init__.py @@ -9,10 +9,10 @@ Modified-History: 2022-09-13, Bo Zhang, fixed a bug """ -import os - from .status import CsstResult, CsstStatus +from .pipeline import Pipeline +from .ccds import CCDS +from .dfs import DFS +from .decorator import parameterized_module_decorator -__version__ = "0.0.1" - -PACKAGE_PATH = os.path.dirname(__file__) +__version__ = "0.0.2" diff --git a/csst_common/ccds/slsconf.py b/csst_common/ccds/slsconf.py new file mode 100644 index 0000000..381a0ea --- /dev/null +++ b/csst_common/ccds/slsconf.py @@ -0,0 +1,350 @@ +import json +import os +import re + +import astropy.io.fits as pyfits + +from ccds import client + + +# chipid: [01, 02, 03, 04, 05, 10, 21, 26, 27, 28, 29, 30] +# filter: [GI, GV, GU, GU, GV, GI, GI, GV, GU, GU, GV, GI] +# clabel: [GI-1, GV-1, GU-1, GU-2, GV-2, GI-2, GI-3, GV-3, GU-3, GU-4, GV-4, GI-4] + + +def get_version(): + observatory = client.get_default_observatory() + operational_context = client.get_default_context(observatory) + ver = re.split(r"[_.]", operational_context)[1] + return ver + + +def resave_sensitivity(dir_save, chipid, flt): + ver = get_version() + h = pyfits.open( + dir_save + "CSST_MSC_MS_SENSITIVITY_" + chipid + "_" + ver + ".fits" + ) + for extname, order in zip( + ["L0ST", "LP1ST", "LM1ST", "LP2ST", "LM2ST"], + ["0st", "1st", "-1st", "2st", "-2st"], + ): + hdu0 = pyfits.PrimaryHDU() + hdu1 = pyfits.BinTableHDU(h[extname].data) + hdul = pyfits.HDUList([hdu0, hdu1]) + hdul.writeto( + dir_save + "CSST_MSC_MS_SENSITIVITY_" + flt + "_" + order + ".fits", + overwrite=True, + checksum=True, + ) + + +def readjson(dir_save, fjsoname): + with open(dir_save + fjsoname) as f: + d = json.load(f) + return d + + +################################################################################## +################################################################################## + +# keys = ['BEAMA', 'MMAG_EXTRACT_A', 'MMAG_MARK_A', 'DYDX_ORDER_A', 'DYDX_A_0', 'DYDX_A_1', 'XOFF_A', 'YOFF_A', +# 'DISP_ORDER_A', 'DLDP_A_0', 'DLDP_A_1', 'BEAMB', 'MMAG_EXTRACT_B', 'MMAG_MARK_B', 'DYDX_ORDER_B', 'DYDX_B_0', +# 'XOFF_B', 'YOFF_B', 'DISP_ORDER_B', 'DLDP_B_0', 'DLDP_B_1', 'BEAMC', 'MMAG_EXTRACT_C', 'MMAG_MARK_C', +# 'DYDX_ORDER_C', 'DYDX_C_0', 'DYDX_C_1', 'XOFF_C', 'YOFF_C', 'DISP_ORDER_C', 'DLDP_C_0', 'DLDP_C_1', 'BEAMD', +# 'MMAG_EXTRACT_D', 'MMAG_MARK_D', 'DYDX_ORDER_D', 'DYDX_D_0', 'DYDX_D_1', 'XOFF_D', 'YOFF_D', 'DISP_ORDER_D', +# 'DLDP_D_0', 'DLDP_D_1', 'BEAME', 'MMAG_EXTRACT_E', 'MMAG_MARK_E', 'DYDX_ORDER_E', 'DYDX_E_0', 'DYDX_E_1', +# 'XOFF_E', 'YOFF_E', 'DISP_ORDER_E', 'DLDP_E_0', 'DLDP_E_1'] + +GL = [ + "GI2", + "GV4", + "GU2", + "GU4", + "GV2", + "GI4", + "GI6", + "GV8", + "GU6", + "GU8", + "GV6", + "GI8", +] +GR = [ + "GI1", + "GV3", + "GU1", + "GU3", + "GV1", + "GI3", + "GI5", + "GV7", + "GU5", + "GU7", + "GV5", + "GI7", +] +SEN = ["GI", "GV", "GU", "GU", "GV", "GI", "GI", "GV", "GU", "GU", "GV", "GI"] + + +def fwriteKEY(fsx, i): + fsx.write("INSTRUMENT CSSTSLS" + "\n") + fsx.write("CAMERA " + SEN[i] + "\n") + if SEN[i] == "GI": + fsx.write("WAVELENGTH 6200 10000" + "\n") + elif SEN[i] == "GV": + fsx.write("WAVELENGTH 4000 6200" + "\n") + elif SEN[i] == "GU": + fsx.write("WAVELENGTH 2550 4000" + "\n") + fsx.write("\n" + "SCIENCE_EXT SCI ; Science extension" + "\n") + fsx.write("DQ_EXT DQ ; DQ extension" + "\n") + fsx.write("ERRORS_EXT ERR ; Error extension" + "\n") + fsx.write("FFNAME csstFlat.fits" + "\n") + fsx.write("DQMASK 246 ; 4096 and 512 taken out" + "\n") + fsx.write("\n" + "RDNOISE 5.0" + "\n") + fsx.write("EXPTIME EXPTIME" + "\n") + fsx.write("POBJSIZE 1.0" + "\n") + fsx.write("#SMFACTOR 1.0" + "\n\n") + + +def fwriteBEAM( + dir_save, + fsx, + i, + GRATINGLR, + BEAMX, + MMAG_EXTRACT_X, + MMAG_MARK_X, + DYDX_ORDER_X, + DYDX_X_0, + DYDX_X_1, + XOFF_X, + YOFF_X, + DISP_ORDER_X, + DLDP_X_0, + DLDP_X_1, +): + ver = get_version() + [ + d01_GI21, + d02_GV43, + d03_GU21, + d04_GU43, + d05_GV21, + d10_GI43, + d21_GI65, + d26_GV87, + d27_GU65, + d28_GU87, + d29_GV65, + d30_GI87, + ] = [ + readjson(dir_save, "CSST_MSC_MS_EXTRACT1D_" + chipid + "_" + ver + ".json") + for chipid in [ + "01", + "02", + "03", + "04", + "05", + "10", + "21", + "26", + "27", + "28", + "29", + "30", + ] + ] + d = [ + d01_GI21, + d02_GV43, + d03_GU21, + d04_GU43, + d05_GV21, + d10_GI43, + d21_GI65, + d26_GV87, + d27_GU65, + d28_GU87, + d29_GV65, + d30_GI87, + ] + fsx.write(BEAMX), [ + fsx.write(" " + str(d[i][GRATINGLR][BEAMX][j])) + for j in range(len(d[i][GRATINGLR][BEAMX])) + ], fsx.write("\n") + fsx.write(MMAG_EXTRACT_X + " " + str(d[i][GRATINGLR][MMAG_EXTRACT_X]) + "\n") + fsx.write(MMAG_MARK_X + " " + str(d[i][GRATINGLR][MMAG_MARK_X]) + "\n") + fsx.write("# " + "\n") + fsx.write("# Trace description " + "\n") + fsx.write("# " + "\n") + fsx.write(DYDX_ORDER_X + " " + str(d[i][GRATINGLR][DYDX_ORDER_X]) + "\n") + fsx.write(DYDX_X_0), [ + fsx.write(" " + str(d[i][GRATINGLR][DYDX_X_0][j])) + for j in range(len(d[i][GRATINGLR][DYDX_X_0])) + ], fsx.write("\n") + if BEAMX == "BEAMB": + pass + else: + fsx.write(DYDX_X_1), [ + fsx.write(" " + str(d[i][GRATINGLR][DYDX_X_1][j])) + for j in range(len(d[i][GRATINGLR][DYDX_X_1])) + ], fsx.write("\n") + fsx.write("# " + "\n") + fsx.write("# X and Y Offsets " + "\n") + fsx.write("# " + "\n") + fsx.write(XOFF_X + " " + str(d[i][GRATINGLR][XOFF_X]) + "\n") + fsx.write(YOFF_X + " " + str(d[i][GRATINGLR][YOFF_X]) + "\n") + fsx.write("# " + "\n") + fsx.write("# Dispersion solution " + "\n") + fsx.write("# " + "\n") + fsx.write(DISP_ORDER_X + " " + str(d[i][GRATINGLR][DISP_ORDER_X]) + "\n") + fsx.write(DLDP_X_0), [ + fsx.write(" " + str(d[i][GRATINGLR][DLDP_X_0][j])) + for j in range(len(d[i][GRATINGLR][DLDP_X_0])) + ], fsx.write("\n") + fsx.write(DLDP_X_1), [ + fsx.write(" " + str(d[i][GRATINGLR][DLDP_X_1][j])) + for j in range(len(d[i][GRATINGLR][DLDP_X_1])) + ], fsx.write("\n") + fsx.write("# " + "\n") + if BEAMX == "BEAMA": + ordername = "1st" + fsx.write( + "SENSITIVITY_A CSST_MSC_MS_SENSITIVITY_" + SEN[i] + "_1st.fits" + "\n" + ) + elif BEAMX == "BEAMB": + ordername = "0st" + fsx.write( + "SENSITIVITY_B CSST_MSC_MS_SENSITIVITY_" + SEN[i] + "_0st.fits" + "\n" + ) + elif BEAMX == "BEAMC": + ordername = "-1st" + fsx.write( + "SENSITIVITY_C CSST_MSC_MS_SENSITIVITY_" + SEN[i] + "_-1st.fits" + "\n" + ) + elif BEAMX == "BEAMD": + ordername = "2st" + fsx.write( + "SENSITIVITY_D CSST_MSC_MS_SENSITIVITY_" + SEN[i] + "_2st.fits" + "\n" + ) + elif BEAMX == "BEAME": + ordername = "-2st" + fsx.write( + "SENSITIVITY_E CSST_MSC_MS_SENSITIVITY_" + SEN[i] + "_-2st.fits" + "\n" + ) + fsx.write("# " + "\n" + "\n") + + +def fsave_conf(dir_save, GLR, GRATINGLR, i): + c = dir_save + "CSST_MSC_MS_" + GLR[i] + ".conf" + os.system("> " + c) + fs = open(c, "a") + fwriteKEY(fs, i) + fs.write("# 1 order (BEAM A) *******************" + "\n") + fwriteBEAM( + dir_save, + fs, + i, + GRATINGLR, + "BEAMA", + "MMAG_EXTRACT_A", + "MMAG_MARK_A", + "DYDX_ORDER_A", + "DYDX_A_0", + "DYDX_A_1", + "XOFF_A", + "YOFF_A", + "DISP_ORDER_A", + "DLDP_A_0", + "DLDP_A_1", + ) + fs.write("\n# 0 order (BEAM B) *******************" + "\n") + fwriteBEAM( + dir_save, + fs, + i, + GRATINGLR, + "BEAMB", + "MMAG_EXTRACT_B", + "MMAG_MARK_B", + "DYDX_ORDER_B", + "DYDX_B_0", + "DYDX_B_1", + "XOFF_B", + "YOFF_B", + "DISP_ORDER_B", + "DLDP_B_0", + "DLDP_B_1", + ) + fs.write("\n# -1 order (BEAM C) *******************" + "\n") + fwriteBEAM( + dir_save, + fs, + i, + GRATINGLR, + "BEAMC", + "MMAG_EXTRACT_C", + "MMAG_MARK_C", + "DYDX_ORDER_C", + "DYDX_C_0", + "DYDX_C_1", + "XOFF_C", + "YOFF_C", + "DISP_ORDER_C", + "DLDP_C_0", + "DLDP_C_1", + ) + fs.write("\n# 2 order (BEAM D) *******************" + "\n") + fwriteBEAM( + dir_save, + fs, + i, + GRATINGLR, + "BEAMD", + "MMAG_EXTRACT_D", + "MMAG_MARK_D", + "DYDX_ORDER_D", + "DYDX_D_0", + "DYDX_D_1", + "XOFF_D", + "YOFF_D", + "DISP_ORDER_D", + "DLDP_D_0", + "DLDP_D_1", + ) + fs.write("\n# -2 order (BEAM E) *******************" + "\n") + fwriteBEAM( + dir_save, + fs, + i, + GRATINGLR, + "BEAME", + "MMAG_EXTRACT_E", + "MMAG_MARK_E", + "DYDX_ORDER_E", + "DYDX_E_0", + "DYDX_E_1", + "XOFF_E", + "YOFF_E", + "DISP_ORDER_E", + "DLDP_E_0", + "DLDP_E_1", + ) + fs.close() + + +def get_slsconf(dir_save=".", **kwargs): + """save SLS conf files to `dir_save`""" + + # resave the sensitivity.fits + for chipid, flt in zip(["01", "02", "03"], ["GI", "GV", "GU"]): + resave_sensitivity(dir_save, chipid, flt) + + # save CSST_MSC_MS_*.conf + for i in range(0, 12): + fsave_conf(dir_save, GL, "GRATINGL", i) + fsave_conf(dir_save, GR, "GRATINGR", i) + + # TODO: assert all files are saved correctly, then return + + # TODO: return a dict containing filepath mapping diff --git a/csst_common/coordinates.py b/csst_common/coordinates.py deleted file mode 100644 index 773f3b9..0000000 --- a/csst_common/coordinates.py +++ /dev/null @@ -1,18 +0,0 @@ -from astropy.wcs import WCS - - -def transform_coordinate( - ra: float = 180.000, - dec: float = 0.0, - original_epoch: float = 2016.0, - target_epoch: float = 2020.0, -) -> tuple[float, float]: - """Transform a coordinate from `original_epoch` to `target_epoch`.""" - pass - - -def transform_wcs( - wcs: WCS, original_epoch: float = 2016.0, target_epoch: float = 2020.0 -) -> WCS: - """Transform a wcs from `original_epoch` to `target_epoch`.""" - pass diff --git a/csst_common/crds/slsconf.py b/csst_common/crds/slsconf.py deleted file mode 100644 index 18b9cc6..0000000 --- a/csst_common/crds/slsconf.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import astropy.io.fits as pyfits, numpy as np, time, os, json -from crds import client - - -# chipid: [01, 02, 03, 04, 05, 10, 21, 26, 27, 28, 29, 30] -# filter: [GI, GV, GU, GU, GV, GI, GI, GV, GU, GU, GV, GI] -# clabel: [GI-1, GV-1, GU-1, GU-2, GV-2, GI-2, GI-3, GV-3, GU-3, GU-4, GV-4, GI-4] - -def get_version(): - observatory = client.get_default_observatory() - operational_context = client.get_default_context(observatory) - ver = re.split(r'[_\.]', operational_context)[1] - return ver - -def resave_sensitivity(dir_save, chipid, flt): - ver = get_version() - h = pyfits.open(dir_save+'CSST_MSC_MS_SENSITIVITY_'+chipid+'_'+ver+'.fits') - for extname, order in zip(['L0ST', 'LP1ST', 'LM1ST', 'LP2ST', 'LM2ST'], ['0st', '1st', '-1st', '2st', '-2st']): - hdu0 = pyfits.PrimaryHDU() - hdu1 = pyfits.BinTableHDU(h[extname].data) - hdul = pyfits.HDUList([hdu0, hdu1]) - hdul.writeto(dir_save+'CSST_MSC_MS_SENSITIVITY_'+flt+'_'+order+'.fits', overwrite=True, checksum=True) - -def readjson(dir_save, fjsoname): - with open(dir_save+fjsoname) as f: - d = json.load(f) - return d - - -################################################################################## -################################################################################## - -# keys = ['BEAMA', 'MMAG_EXTRACT_A', 'MMAG_MARK_A', 'DYDX_ORDER_A', 'DYDX_A_0', 'DYDX_A_1', 'XOFF_A', 'YOFF_A', 'DISP_ORDER_A', 'DLDP_A_0', 'DLDP_A_1', 'BEAMB', 'MMAG_EXTRACT_B', 'MMAG_MARK_B', 'DYDX_ORDER_B', 'DYDX_B_0', 'XOFF_B', 'YOFF_B', 'DISP_ORDER_B', 'DLDP_B_0', 'DLDP_B_1', 'BEAMC', 'MMAG_EXTRACT_C', 'MMAG_MARK_C', 'DYDX_ORDER_C', 'DYDX_C_0', 'DYDX_C_1', 'XOFF_C', 'YOFF_C', 'DISP_ORDER_C', 'DLDP_C_0', 'DLDP_C_1', 'BEAMD', 'MMAG_EXTRACT_D', 'MMAG_MARK_D', 'DYDX_ORDER_D', 'DYDX_D_0', 'DYDX_D_1', 'XOFF_D', 'YOFF_D', 'DISP_ORDER_D', 'DLDP_D_0', 'DLDP_D_1', 'BEAME', 'MMAG_EXTRACT_E', 'MMAG_MARK_E', 'DYDX_ORDER_E', 'DYDX_E_0', 'DYDX_E_1', 'XOFF_E', 'YOFF_E', 'DISP_ORDER_E', 'DLDP_E_0', 'DLDP_E_1'] - -GL = ['GI2', 'GV4', 'GU2', 'GU4', 'GV2', 'GI4', 'GI6', 'GV8', 'GU6', 'GU8', 'GV6', 'GI8' ] -GR = ['GI1', 'GV3', 'GU1', 'GU3', 'GV1', 'GI3', 'GI5', 'GV7', 'GU5', 'GU7', 'GV5', 'GI7' ] -SEN = ['GI', 'GV', 'GU', 'GU', 'GV', 'GI', 'GI', 'GV', 'GU', 'GU', 'GV', 'GI' ] - -def fwriteKEY(fsx, i): - fsx.write('INSTRUMENT CSSTSLS'+'\n') - fsx.write('CAMERA '+SEN[i]+'\n') - if SEN[i] == 'GI': - fsx.write('WAVELENGTH 6200 10000'+'\n') - elif SEN[i] == 'GV': - fsx.write('WAVELENGTH 4000 6200'+'\n') - elif SEN[i] == 'GU': - fsx.write('WAVELENGTH 2550 4000'+'\n') - fsx.write('\n'+'SCIENCE_EXT SCI ; Science extension'+'\n') - fsx.write('DQ_EXT DQ ; DQ extension'+'\n') - fsx.write('ERRORS_EXT ERR ; Error extension'+'\n') - fsx.write('FFNAME csstFlat.fits'+'\n') - fsx.write('DQMASK 246 ; 4096 and 512 taken out'+'\n') - fsx.write('\n'+'RDNOISE 5.0'+'\n') - fsx.write('EXPTIME EXPTIME'+'\n') - fsx.write('POBJSIZE 1.0'+'\n') - fsx.write('#SMFACTOR 1.0'+'\n\n') - -def fwriteBEAM(dir_save, fsx, i, GRATINGLR, BEAMX, MMAG_EXTRACT_X, MMAG_MARK_X, DYDX_ORDER_X, DYDX_X_0, DYDX_X_1, XOFF_X, YOFF_X, DISP_ORDER_X, DLDP_X_0, DLDP_X_1): - ver = get_version() - [d01_GI21, d02_GV43, d03_GU21, d04_GU43, d05_GV21, d10_GI43, d21_GI65, d26_GV87, d27_GU65, d28_GU87, d29_GV65, d30_GI87] = [readjson(dir_save, 'CSST_MSC_MS_EXTRACT1D_'+chipid+'_'+ver+'.json') for chipid in ['01', '02', '03', '04', '05', '10', '21', '26', '27', '28', '29', '30']] - d = [d01_GI21, d02_GV43, d03_GU21, d04_GU43, d05_GV21, d10_GI43, d21_GI65, d26_GV87, d27_GU65, d28_GU87, d29_GV65, d30_GI87] - fsx.write(BEAMX), [fsx.write(' '+str(d[i][GRATINGLR][BEAMX][j])) for j in range(len(d[i][GRATINGLR][BEAMX]))], fsx.write('\n') - fsx.write(MMAG_EXTRACT_X+' '+str(d[i][GRATINGLR][MMAG_EXTRACT_X])+'\n') - fsx.write(MMAG_MARK_X+' '+str(d[i][GRATINGLR][MMAG_MARK_X])+'\n') - fsx.write('# '+'\n') - fsx.write('# Trace description '+'\n') - fsx.write('# '+'\n') - fsx.write(DYDX_ORDER_X+' '+str(d[i][GRATINGLR][DYDX_ORDER_X])+'\n') - fsx.write(DYDX_X_0), [fsx.write(' '+str(d[i][GRATINGLR][DYDX_X_0][j])) for j in range(len(d[i][GRATINGLR][DYDX_X_0]))], fsx.write('\n') - if BEAMX == 'BEAMB': - pass - else: - fsx.write(DYDX_X_1), [fsx.write(' '+str(d[i][GRATINGLR][DYDX_X_1][j])) for j in range(len(d[i][GRATINGLR][DYDX_X_1]))], fsx.write('\n') - fsx.write('# '+'\n') - fsx.write('# X and Y Offsets '+'\n') - fsx.write('# '+'\n') - fsx.write(XOFF_X+' '+str(d[i][GRATINGLR][XOFF_X])+'\n') - fsx.write(YOFF_X+' '+str(d[i][GRATINGLR][YOFF_X])+'\n') - fsx.write('# '+'\n') - fsx.write('# Dispersion solution '+'\n') - fsx.write('# '+'\n') - fsx.write(DISP_ORDER_X+' '+str(d[i][GRATINGLR][DISP_ORDER_X])+'\n') - fsx.write(DLDP_X_0), [fsx.write(' '+str(d[i][GRATINGLR][DLDP_X_0][j])) for j in range(len(d[i][GRATINGLR][DLDP_X_0]))], fsx.write('\n') - fsx.write(DLDP_X_1), [fsx.write(' '+str(d[i][GRATINGLR][DLDP_X_1][j])) for j in range(len(d[i][GRATINGLR][DLDP_X_1]))], fsx.write('\n') - fsx.write('# '+'\n') - if BEAMX == 'BEAMA': - ordername = '1st' - fsx.write('SENSITIVITY_A CSST_MSC_MS_SENSITIVITY_'+SEN[i]+'_1st.fits'+'\n') - elif BEAMX == 'BEAMB': - ordername = '0st' - fsx.write('SENSITIVITY_B CSST_MSC_MS_SENSITIVITY_'+SEN[i]+'_0st.fits'+'\n') - elif BEAMX == 'BEAMC': - ordername = '-1st' - fsx.write('SENSITIVITY_C CSST_MSC_MS_SENSITIVITY_'+SEN[i]+'_-1st.fits'+'\n') - elif BEAMX == 'BEAMD': - ordername = '2st' - fsx.write('SENSITIVITY_D CSST_MSC_MS_SENSITIVITY_'+SEN[i]+'_2st.fits'+'\n') - elif BEAMX == 'BEAME': - ordername = '-2st' - fsx.write('SENSITIVITY_E CSST_MSC_MS_SENSITIVITY_'+SEN[i]+'_-2st.fits'+'\n') - fsx.write('# '+'\n'+'\n') - - -def fsave_conf(dir_save, GLR, GRATINGLR, i): - c = dir_save+'CSST_MSC_MS_'+GLR[i]+'.conf' - os.system('> '+c) - fs = open(c, 'a') - fwriteKEY(fs, i) - fs.write('# 1 order (BEAM A) *******************'+'\n') - fwriteBEAM(dir_save, fs, i, GRATINGLR, 'BEAMA', 'MMAG_EXTRACT_A', 'MMAG_MARK_A', 'DYDX_ORDER_A', 'DYDX_A_0', 'DYDX_A_1', 'XOFF_A', 'YOFF_A', 'DISP_ORDER_A', 'DLDP_A_0', 'DLDP_A_1') - fs.write('\n# 0 order (BEAM B) *******************'+'\n') - fwriteBEAM(dir_save, fs, i, GRATINGLR, 'BEAMB', 'MMAG_EXTRACT_B', 'MMAG_MARK_B', 'DYDX_ORDER_B', 'DYDX_B_0', 'DYDX_B_1', 'XOFF_B', 'YOFF_B', 'DISP_ORDER_B', 'DLDP_B_0', 'DLDP_B_1') - fs.write('\n# -1 order (BEAM C) *******************'+'\n') - fwriteBEAM(dir_save, fs, i, GRATINGLR, 'BEAMC', 'MMAG_EXTRACT_C', 'MMAG_MARK_C', 'DYDX_ORDER_C', 'DYDX_C_0', 'DYDX_C_1', 'XOFF_C', 'YOFF_C', 'DISP_ORDER_C', 'DLDP_C_0', 'DLDP_C_1') - fs.write('\n# 2 order (BEAM D) *******************'+'\n') - fwriteBEAM(dir_save, fs, i, GRATINGLR, 'BEAMD', 'MMAG_EXTRACT_D', 'MMAG_MARK_D', 'DYDX_ORDER_D', 'DYDX_D_0', 'DYDX_D_1', 'XOFF_D', 'YOFF_D', 'DISP_ORDER_D', 'DLDP_D_0', 'DLDP_D_1') - fs.write('\n# -2 order (BEAM E) *******************'+'\n') - fwriteBEAM(dir_save, fs, i, GRATINGLR, 'BEAME', 'MMAG_EXTRACT_E', 'MMAG_MARK_E', 'DYDX_ORDER_E', 'DYDX_E_0', 'DYDX_E_1', 'XOFF_E', 'YOFF_E', 'DISP_ORDER_E', 'DLDP_E_0', 'DLDP_E_1') - fs.close() - - -def get_slsconf(dir_save=".", **kwargs): - """ save SLS conf files to `dir_save` """ - - # resave the sensitivity.fits - for chipid, flt in zip(['01', '02', '03'], ['GI', 'GV', 'GU']): - resave_sensitivity(dir_save,chipid, flt) - - # save CSST_MSC_MS_*.conf - for i in range(0,12): - fsave_conf(dir_save, GL, 'GRATINGL', i) - fsave_conf(dir_save, GR, 'GRATINGR', i) - - pass - diff --git a/csst_common/data/node_conf.yml b/csst_common/data/node_conf.yml deleted file mode 100644 index bb7ee74..0000000 --- a/csst_common/data/node_conf.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -pml: - CSST_DFS_API_MODE: "cluster" - CSST_DFS_GATEWAY: "172.24.27.2:30880" - CSST_DFS_APP_ID: "test" - CSST_DFS_APP_TOKEN: "test" - -kmust: - CSST_DFS_API_MODE: "cluster" - CSST_DFS_GATEWAY: "222.197.214.168:30880" - CSST_DFS_APP_ID: "1" - CSST_DFS_APP_TOKEN: "1" - -tcc: - CSST_DFS_API_MODE: "cluster" - CSST_DFS_GATEWAY: "10.0.0.8:30880" - CSST_DFS_APP_ID: "test" - CSST_DFS_APP_TOKEN: "test" diff --git a/csst_common/data_manager.py b/csst_common/data_manager.py index 264e6fc..6cc9b47 100644 --- a/csst_common/data_manager.py +++ b/csst_common/data_manager.py @@ -32,9 +32,8 @@ from csst_dfs_api.sls.level2spectra import Level2SpectraApi as SlsLevel2DataApi from .logger import get_logger from .params import CSST_PARAMS as CP -from .params import DFS_CONF from .time import now -from .crds import CRDS +from .ccds import CCDS from .dfs import DFS @@ -89,11 +88,11 @@ class CsstMsDataManager: dfs_root : str The DFS root path. telescope : str - The telescope name. Defaults to ``CSST`` for C5.2 simulation. + The telescope pipeline_id. Defaults to ``CSST`` for C5.2 simulation. instrument : str - The instrument name. Defaults to ``MSC`` for C5.2 simulation. + The instrument pipeline_id. Defaults to ``MSC`` for C5.2 simulation. project : str - The survey name. Defaults to ``MS`` for C5.2 simulation. + The survey pipeline_id. Defaults to ``MS`` for C5.2 simulation. obs_type : str The image type signature for science images. Defualts to ``SCI`` for C5.2 simulation. exp_start : int @@ -107,11 +106,11 @@ class CsstMsDataManager: version : str The L0 version. ext : str - The extension name, i.e., ``.fits``. + The extension pipeline_id, i.e., ``.fits``. log_ppl : str - The pipeline log file name. + The pipeline log file pipeline_id. log_mod : str - The module log file name. + The module log file pipeline_id. clear_dir : bool If True, clear ``dm.dir_l1`` directory. verbose : bool @@ -148,36 +147,36 @@ class CsstMsDataManager: """ # TODO: update docstring - def __init__(self, - ver_sim: str = "C6.2", - datatype: str = "mbi", - available_detectors: Union[None, list] = None, - target_detectors: Union[None, list, int] = None, - dir_l0: str = ".", - dir_l1: str = ".", - path_aux: str = "/pipeline/aux", # aux dir - dfs_root: str = "/dfsroot", - telescope: str = "CSST", - instrument: str = "MSC", - project: str = "MS", - obs_type: str = "SCI", - exp_start: int = "20270810081950", - exp_stop: int = "20270810082220", - obs_id: str = "100000100", - level: str = "0", - version: str = "01", - ext: str = "fits", - log_ppl: str = "", - log_mod: str = "", - clear_dir: bool = False, - verbose: bool = True, - n_jobs: int = 18, - n_jobs_gpu: int = 1, - backend: str = "multiprocessing", - device: str = "CPU", - stamps: str = "", - ): - + def __init__( + self, + ver_sim: str = "C6.2", + datatype: str = "mbi", + available_detectors: Union[None, list] = None, + target_detectors: Union[None, list, int] = None, + dir_l0: str = ".", + dir_l1: str = ".", + path_aux: str = "/pipeline/aux", # aux dir + dfs_root: str = "/dfsroot", + telescope: str = "CSST", + instrument: str = "MSC", + project: str = "MS", + obs_type: str = "SCI", + exp_start: int = "20270810081950", + exp_stop: int = "20270810082220", + obs_id: str = "100000100", + level: str = "0", + version: str = "01", + ext: str = "fits", + log_ppl: str = "", + log_mod: str = "", + clear_dir: bool = False, + verbose: bool = True, + n_jobs: int = 18, + n_jobs_gpu: int = 1, + backend: str = "multiprocessing", + device: str = "CPU", + stamps: str = "", + ): # set DFS log dir os.environ["CSST_DFS_LOGS_DIR"] = "." @@ -205,7 +204,9 @@ class CsstMsDataManager: print("Valid detectors are: ", self.valid_detectors) # available_detectors - self.available_detectors = available_detectors if available_detectors is not None else list() + self.available_detectors = ( + available_detectors if available_detectors is not None else list() + ) if verbose: print("Available detectors are:", self.available_detectors) # set all available detectors by default @@ -218,7 +219,7 @@ class CsstMsDataManager: self.exp_start = exp_start self.exp_stop = exp_stop - # file name components + # file pipeline_id components self.telescope = telescope self.instrument = instrument self.project = project @@ -254,28 +255,46 @@ class CsstMsDataManager: self.clear_dir(self.dir_l1) # L1 whitelist - self.l1_whitelist = [self.l1_file(name="mrs.csv"), ] + self.l1_whitelist = [ + self.l1_file(name="mrs.csv"), + ] for detector in self.target_detectors: if self.datatype == "mbi": - self.l1_whitelist.append(self.l1_detector(detector=detector, post="img_L1.fits")) - self.l1_whitelist.append(self.l1_detector(detector=detector, post="wht_L1.fits")) - self.l1_whitelist.append(self.l1_detector(detector=detector, post="flg_L1.fits")) - self.l1_whitelist.append(self.l1_detector(detector=detector, post="cat.fits")) - self.l1_whitelist.append(self.l1_detector(detector=detector, post="psf.fits")) + self.l1_whitelist.append( + self.l1_detector(detector=detector, post="img_L1.fits") + ) + self.l1_whitelist.append( + self.l1_detector(detector=detector, post="wht_L1.fits") + ) + self.l1_whitelist.append( + self.l1_detector(detector=detector, post="flg_L1.fits") + ) + self.l1_whitelist.append( + self.l1_detector(detector=detector, post="cat.fits") + ) + self.l1_whitelist.append( + self.l1_detector(detector=detector, post="psf.fits") + ) elif self.datatype == "sls": - self.l1_whitelist.append(self.l1_detector(detector=detector, post="L1_1.fits")) + self.l1_whitelist.append( + self.l1_detector(detector=detector, post="L1_1.fits") + ) # pipeline logger if log_ppl == "": self.logger_ppl = get_logger(name="CSST L1 Pipeline Logger", filename="") else: - self.logger_ppl = get_logger(name="CSST L1 Pipeline Logger", filename=os.path.join(dir_l1, log_ppl)) + self.logger_ppl = get_logger( + name="CSST L1 Pipeline Logger", filename=os.path.join(dir_l1, log_ppl) + ) self.l1_whitelist.append(os.path.join(dir_l1, log_ppl)) # module logger if log_mod == "": self.logger_mod = get_logger(name="CSST L1 Module Logger", filename="") else: - self.logger_mod = get_logger(name="CSST L1 Module Logger", filename=os.path.join(dir_l1, log_mod)) + self.logger_mod = get_logger( + name="CSST L1 Module Logger", filename=os.path.join(dir_l1, log_mod) + ) self.l1_whitelist.append(os.path.join(dir_l1, log_mod)) self.custom_bias = None @@ -336,31 +355,41 @@ class CsstMsDataManager: def target_detectors(self, detectors: Union[None, list, int] = None): assert detectors is None or type(detectors) in [list, int] if detectors is None: - self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors)) + self._target_detectors = list( + set(self.available_detectors) & set(self.valid_detectors) + ) elif isinstance(detectors, list): - self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & set(detectors)) + self._target_detectors = list( + set(self.available_detectors) + & set(self.valid_detectors) + & set(detectors) + ) elif isinstance(detectors, int): - self._target_detectors = list(set(self.available_detectors) & set(self.valid_detectors) & {detectors}) + self._target_detectors = list( + set(self.available_detectors) & set(self.valid_detectors) & {detectors} + ) def set_detectors(self, detectors=None): - raise DeprecationWarning("This method is deprecated, please directly use dm.target_detectors = detectors!") + raise DeprecationWarning( + "This method is deprecated, please directly use dm.target_detectors = detectors!" + ) @staticmethod def from_dir( - ver_sim="C6.2", - datatype="mbi", - dir_l0=".", - dir_l1=".", - path_aux="", - pattern="CSST_MSC_*_SCIE_*.fits", - log_ppl="csst-l1ppl.log", - log_mod="csst-l1mod.log", - n_jobs=18, - backend="multiprocessing", - device="CPU", - **kwargs + ver_sim="C6.2", + datatype="mbi", + dir_l0=".", + dir_l1=".", + path_aux="", + pattern="CSST_MSC_*_SCIE_*.fits", + log_ppl="csst-l1ppl.log", + log_mod="csst-l1mod.log", + n_jobs=18, + backend="multiprocessing", + device="CPU", + **kwargs, ): - """ initialize the multi-band imaging data manager """ + """initialize the multi-band imaging data manager""" assert ver_sim in CP["sim"]["versions"] @@ -408,7 +437,9 @@ class CsstMsDataManager: mogd.pop("detector") # available detectors - available_detectors = [int(re.fullmatch(pattern, fp)["detector"]) for fp in fps_img] + available_detectors = [ + int(re.fullmatch(pattern, fp)["detector"]) for fp in fps_img + ] available_detectors.sort() return CsstMsDataManager( @@ -425,12 +456,12 @@ class CsstMsDataManager: backend=backend, device=device, **mogd, - **kwargs + **kwargs, ) @staticmethod def glob_image(dir_l0, pattern="CSST_MSC_*_SCIE_*.fits"): - """ glob files in L0 data directory """ + """glob files in L0 data directory""" fps = glob.glob(os.path.join(dir_l0, pattern)) fps = [os.path.basename(fp) for fp in fps] fps.sort() @@ -438,11 +469,11 @@ class CsstMsDataManager: return fps def l0_id(self, detector=6): - """ Level0 ID, consistent with DFS. """ + """Level0 ID, consistent with DFS.""" return f"{self.obs_id}{detector:02d}" def l0_cat(self, detector=6): - """ the L0 cat file path""" + """the L0 cat file path""" assert detector in self.available_detectors fn = f"{self.instrument}_{self.obs_id}_chip_{detector:02d}_filt_{self.detector2filter[detector]}.cat" fp = os.path.join(self.dir_l0, fn) @@ -451,7 +482,7 @@ class CsstMsDataManager: return fp def l0_log(self, detector=6): - """ L0 log file path """ + """L0 log file path""" assert detector in self.available_detectors fn = f"{self.instrument}_{self.obs_id}_chip_{detector:02d}_filt_{self.detector2filter[detector]}.log" fp = os.path.join(self.dir_l0, fn) @@ -460,35 +491,43 @@ class CsstMsDataManager: return fp def l0_detector(self, detector=6): - """ L0 detector-specific image file path """ + """L0 detector-specific image file path""" assert detector in self.available_detectors if self.ver_sim in ["C5.2", "C6.1"]: - fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" \ - f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L{self.level}_{self.version}.{self.ext}" + fn = ( + f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L{self.level}_{self.version}.{self.ext}" + ) else: - fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" \ - f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L{self.level}_V{self.version}.{self.ext}" + fn = ( + f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L{self.level}_V{self.version}.{self.ext}" + ) fp = os.path.join(self.dir_l0, fn) if not os.path.exists(fp): raise FileNotFoundError(f"File not found: {fp}") return fp def l0_crs(self, detector=6): - """ L0 cosmic ray file path """ + """L0 cosmic ray file path""" assert detector in self.available_detectors if self.ver_sim == "C5.2": - fn = f"{self.telescope}_{self.instrument}_{self.project}_CRS_{self.exp_start}_" \ - f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L{self.level}_{self.version}.{self.ext}" + fn = ( + f"{self.telescope}_{self.instrument}_{self.project}_CRS_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L{self.level}_{self.version}.{self.ext}" + ) else: - fn = f"{self.telescope}_{self.instrument}_{self.project}_CRS_{self.exp_start}_" \ - f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L{self.level}_V{self.version}.{self.ext}" + fn = ( + f"{self.telescope}_{self.instrument}_{self.project}_CRS_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L{self.level}_V{self.version}.{self.ext}" + ) fp = os.path.join(self.dir_l0, fn) if not os.path.exists(fp): raise FileNotFoundError(f"File not found: {fp}") return fp def l1_detector(self, detector=6, post="IMG.fits"): - """ generate L1 file path + """generate L1 file path Parameters ---------- @@ -506,11 +545,15 @@ class CsstMsDataManager: assert detector in self.available_detectors if post == ".fits": # no additional suffix - fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" - f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L1_V{self.version}{post}") + fn = ( + f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L1_V{self.version}{post}" + ) else: - fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" - f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L1_V{self.version}_{post}") + fn = ( + f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{detector:02d}_L1_V{self.version}_{post}" + ) return os.path.join(self.dir_l1, fn) def get_refs(self, detector=6): @@ -523,17 +566,16 @@ class CsstMsDataManager: Returns ------- dict - {'bias': '/crdsroot/references/msc/csst_msc_ms_bias_11_000001.fits', - 'dark': '/crdsroot/references/msc/csst_msc_ms_dark_11_000001.fits', - 'ledflat': '/crdsroot/references/msc/csst_msc_ms_ledflat_11_000001.fits', - 'shutter': '/crdsroot/references/msc/csst_msc_ms_shutter_11_000001.fits'} + {'bias': '/ccds_root/references/msc/csst_msc_ms_bias_11_000001.fits', + 'dark': '/ccds_root/references/msc/csst_msc_ms_dark_11_000001.fits', + 'ledflat': '/ccds_root/references/msc/csst_msc_ms_ledflat_11_000001.fits', + 'shutter': '/ccds_root/references/msc/csst_msc_ms_shutter_11_000001.fits'} """ if self.crds.is_available: try: print("CRDS available, use refs from CRDS ...") return self.crds.retry( - self.crds.get_refs, 3, - file_path=self.l0_detector(detector) + self.crds.get_refs, 3, file_path=self.l0_detector(detector) ) except BaseException as e: print("CRDS reference access failed! ", e) @@ -548,18 +590,18 @@ class CsstMsDataManager: fp = os.path.join( self.path_aux, "C6.2_ref_crds", - "csst_msc_ms_{}_{:02d}_{:06d}.fits".format(ref_type, detector, 1) + "csst_msc_ms_{}_{:02d}_{:06d}.fits".format(ref_type, detector, 1), ) assert os.path.exists(fp), f"File not found: [{fp}]" refs[ref_type] = fp return refs def get_bias(self, detector=6): - """ get bias data """ + """get bias data""" fp = os.path.join( self.path_aux, "C6.2_ref_crds", - "csst_msc_ms_{}_{:02d}_{:06d}.fits".format("bias", detector, 1) + "csst_msc_ms_{}_{:02d}_{:06d}.fits".format("bias", detector, 1), ) if not os.path.exists(fp): raise FileNotFoundError(fp) @@ -567,11 +609,11 @@ class CsstMsDataManager: return fp def get_dark(self, detector=6): - """ get dark data """ + """get dark data""" fp = os.path.join( self.path_aux, "C6.2_ref_crds", - "csst_msc_ms_{}_{:02d}_{:06d}.fits".format("dark", detector, 1) + "csst_msc_ms_{}_{:02d}_{:06d}.fits".format("dark", detector, 1), ) if not os.path.exists(fp): raise FileNotFoundError(fp) @@ -579,11 +621,11 @@ class CsstMsDataManager: return fp def get_flat(self, detector=6): - """ get flat data """ + """get flat data""" fp = os.path.join( self.path_aux, "C6.2_ref_crds", - "csst_msc_ms_{}_{:02d}_{:06d}.fits".format("ledflat", detector, 1) + "csst_msc_ms_{}_{:02d}_{:06d}.fits".format("ledflat", detector, 1), ) if not os.path.exists(fp): raise FileNotFoundError(fp) @@ -591,11 +633,11 @@ class CsstMsDataManager: return fp def get_shutter(self, detector=6): - """ get flat data """ + """get flat data""" fp = os.path.join( self.path_aux, "C6.2_ref_crds", - "csst_msc_ms_{}_{:02d}_{:06d}.fits".format("shutter", detector, 1) + "csst_msc_ms_{}_{:02d}_{:06d}.fits".format("shutter", detector, 1), ) if not os.path.exists(fp): raise FileNotFoundError(fp) @@ -606,14 +648,14 @@ class CsstMsDataManager: # return os.path.join(self.path_aux, "axeconf") # "/home/csstpipeline/L1Pipeline/aux/axeconf" def l1_file(self, name="", comment=""): - """ L1 file path + """L1 file path Parameters ---------- name : str - file name + file pipeline_id comment : str - use the function name plz + use the function pipeline_id plz Returns ------- @@ -627,7 +669,7 @@ class CsstMsDataManager: return fp def get_sls_info(self): - """ Get the target SLS image header info and return. """ + """Get the target SLS image header info and return.""" # if self.use_dfs: # raise NotImplementedError() # else: @@ -636,17 +678,23 @@ class CsstMsDataManager: return header def get_mbi_info(self): - """ Get all MBI image header info and return as a table. """ + """Get all MBI image header info and return as a table.""" # if self.use_dfs: # raise NotImplementedError() # else: - info = Table.read("/nfsdata/share/csst_simulation_data/Cycle-5-SimuData/slitlessSpectroscopy/t_mbi_l1.fits") + info = Table.read( + "/nfsdata/share/csst_simulation_data/Cycle-5-SimuData/slitlessSpectroscopy/t_mbi_l1.fits" + ) return info @staticmethod def quickstart( - ver_sim="C6.2", datatype="mbi", dir_l1=".", exposure_id=100, - n_jobs=18, backend="multiprocessing" + ver_sim="C6.2", + datatype="mbi", + dir_l1=".", + exposure_id=100, + n_jobs=18, + backend="multiprocessing", ): """ Quick dataset generator for tests on dandelion or PML @@ -677,16 +725,24 @@ class CsstMsDataManager: raise ValueError("Please use ver_sim = 'C6.2'! and exposure_id = 100") assert datatype in ["mbi", "sls"] - # auto identify node name + # auto identify node pipeline_id hostname = os.uname()[1] - assert hostname in ["dandelion", "tulip", ] + assert hostname in [ + "dandelion", + "tulip", + ] dir_l0 = "/nfsdata/share/pipeline-unittest/MSC_C6.2_UNITTEST/MSC_0000100" path_aux = "/nfsdata/share/pipeline/aux" return CsstMsDataManager.from_dir( - ver_sim=ver_sim, datatype=datatype, dir_l0=dir_l0, dir_l1=dir_l1, path_aux=path_aux, - n_jobs=n_jobs, backend=backend + ver_sim=ver_sim, + datatype=datatype, + dir_l0=dir_l0, + dir_l1=dir_l1, + path_aux=path_aux, + n_jobs=n_jobs, + backend=backend, ) def __repr__(self): @@ -703,11 +759,11 @@ class CsstMsDataManager: return lines def remove_files(self, fmt="*.fits"): - """ Remove L1 files conforming the format. """ + """Remove L1 files conforming the format.""" os.system(f"rm -rf {os.path.join(self.dir_l1, fmt)}") def remove_dir(self, dir_name): - """ Remove L1 (sub-)directory. """ + """Remove L1 (sub-)directory.""" os.system(f"rm -rf {os.path.join(self.dir_l1, dir_name)}") @staticmethod @@ -721,25 +777,25 @@ class CsstMsDataManager: @staticmethod def dfs_is_available(): - """ Test if DFS works. """ + """Test if DFS works.""" try: tbl = CatalogApi().catalog_query( catalog_name="gaia3", ra=180, dec=0, - radius=.1, + radius=0.1, columns=("ra", "dec"), min_mag=0, max_mag=30, obstime=-1, - limit=-1 + limit=-1, ) return len(tbl) > 0 except: return False def get_coord(self): - """ Get pointing coordinate. """ + """Get pointing coordinate.""" header = fits.getheader(self.l0_detector(detector=self.target_detectors[0])) if self.ver_sim == "C5.2": pointing_ra = header["RA_OBJ"] @@ -750,7 +806,7 @@ class CsstMsDataManager: return pointing_ra, pointing_dec def dfs_rc_auto(self): - """ Download RC for the current exposure. """ + """Download RC for the current exposure.""" assert self.dfs_is_available() assert len(self.target_detectors) >= 1 pointing_ra, pointing_dec = self.get_coord() @@ -758,54 +814,54 @@ class CsstMsDataManager: ra=pointing_ra, dec=pointing_dec, columns=( - 'ref_epoch', - 'ra', - 'ra_error', - 'dec', - 'dec_error', - 'parallax', - 'parallax_error', - 'pmra', - 'pmra_error', - 'pmdec', - 'pmdec_error', - 'phot_g_mean_mag', - 'source_id' + "ref_epoch", + "ra", + "ra_error", + "dec", + "dec_error", + "parallax", + "parallax_error", + "pmra", + "pmra_error", + "pmdec", + "pmdec_error", + "phot_g_mean_mag", + "source_id", ), radius=2, min_mag=0, max_mag=30, obstime=-1, - limit=-1 + limit=-1, ) return refcat def dfs_rc_query( - self, - ra=180, - dec=0, - radius=2, - columns=( - 'ref_epoch', - 'ra', - 'ra_error', - 'dec', - 'dec_error', - 'parallax', - 'parallax_error', - 'pmra', - 'pmra_error', - 'pmdec', - 'pmdec_error', - 'phot_g_mean_mag', - 'source_id' - ), - min_mag=0, - max_mag=30, - obstime=-1, - limit=-1 + self, + ra=180, + dec=0, + radius=2, + columns=( + "ref_epoch", + "ra", + "ra_error", + "dec", + "dec_error", + "parallax", + "parallax_error", + "pmra", + "pmra_error", + "pmdec", + "pmdec_error", + "phot_g_mean_mag", + "source_id", + ), + min_mag=0, + max_mag=30, + obstime=-1, + limit=-1, ): - """ Query Reference Catalog (RC) from DFS. + """Query Reference Catalog (RC) from DFS. Ref. https://gea.esac.esa.int/archive/documentation/GDR3/Gaia_archive/chap_datamodel/ sec_dm_main_source_catalogue/ssec_dm_gaia_source.html @@ -819,13 +875,13 @@ class CsstMsDataManager: min_mag=min_mag, max_mag=max_mag, obstime=obstime, - limit=limit + limit=limit, ) if cat["code"] == 0: self.logger_ppl.info( f"Results from DFS CATAPI:\n" f" - code = {cat['code']}\n" - f" - message = {cat['message']}\n" + f" - msg = {cat['msg']}\n" f" - totalCount = {cat['totalCount']}\n" f" - columns = {cat['columns']}" ) @@ -834,30 +890,30 @@ class CsstMsDataManager: self.logger_ppl.info( f"Results from DFS CATAPI:\n" f" - code = {cat['code']}\n" - f" - message = {cat['message']}" + f" - msg = {cat['msg']}" ) raise ValueError("Bad catalog query result!") def dfs_l1_push(self): - """ Push MBI/SLS L1 data to DFS. """ + """Push MBI/SLS L1 data to DFS.""" # l1api = get_l1api() # l1api.write() return def dfs_l2_push(self): - """ Push SLS spectra to DFS. """ + """Push SLS spectra to DFS.""" pass @staticmethod def from_l1id( - l1_id="1000000001", - datatype="sls", - dir_l0="/L1Pipeline/L0", - dir_l1="/L1Pipeline/L1", - use_dfs=True, - dfs_node="pml", - clear_l1=False, - dfs_root="/share/dfs" + l1_id="1000000001", + datatype="sls", + dir_l0="/L1Pipeline/L0", + dir_l1="/L1Pipeline/L1", + use_dfs=True, + dfs_node="pml", + clear_l1=False, + dfs_root="/share/dfs", ): pass @@ -866,7 +922,7 @@ class CsstMsDataManager: # query for L0 data print(f"Query obs_id={obs_id} ...", end="") recs = Level0DataApi().find(obs_id=obs_id) - print("Message: \n", recs) + print("MessageWriter: \n", recs) print(f"{recs['totalCount']} records obtained!") assert recs["code"] == 0 assert recs["totalCount"] > 0 @@ -883,20 +939,20 @@ class CsstMsDataManager: @staticmethod def from_dfs( - obs_id="100000100", - ver_sim="C6.2", - datatype="mbi", - dir_l0="/pipeline/L0", - dir_l1="/pipeline/L1", - path_aux="/pipeline/aux", - clear_l1=False, - dfs_root="/share/dfs", - n_jobs=18, - backend="multiprocessing", - device="CPU", - **kwargs + obs_id="100000100", + ver_sim="C6.2", + datatype="mbi", + dir_l0="/pipeline/L0", + dir_l1="/pipeline/L1", + path_aux="/pipeline/aux", + clear_l1=False, + dfs_root="/share/dfs", + n_jobs=18, + backend="multiprocessing", + device="CPU", + **kwargs, ): - """ Initialize CsstMsDataManager from DFS. """ + """Initialize CsstMsDataManager from DFS.""" # (clear and) make directories if os.path.exists(dir_l0): os.system(f"rm -rf {dir_l0}/*") @@ -936,14 +992,14 @@ class CsstMsDataManager: n_jobs=n_jobs, backend=backend, device=device, - **kwargs + **kwargs, ) assert dm.obs_id == obs_id return dm def dfs_l0_query(self, obs_id: str = "100000100"): - """ Query L0 data from DFS. """ + """Query L0 data from DFS.""" result = self.dfs_L0DataApi.find(obs_id=str(obs_id)) print(f"{result['totalCount']} records returned from DFS.") if not result["code"] == 0: @@ -953,24 +1009,36 @@ class CsstMsDataManager: # Check if all 30 detectors are available for detector in CP["all"]["detectors"]: for obs_type in ["sci", "cosmic_ray"]: - if np.sum((tbl["detector_no"] == f"{detector:02d}") & (tbl["obs_type"] == obs_type)) == 0: - self.logger_ppl.warning(f"Record not found for detector {detector:02d} and obs_type {obs_type}") + if ( + np.sum( + (tbl["detector_no"] == f"{detector:02d}") + & (tbl["obs_type"] == obs_type) + ) + == 0 + ): + self.logger_ppl.warning( + f"Record not found for detector {detector:02d} and obs_type {obs_type}" + ) return tbl def dfs_l0_check_all(self): - """ Check all C5.2 L0 data is available in DFS. """ + """Check all C5.2 L0 data is available in DFS.""" is_good = True for obs_id in range(100000020, 100000155): tbl = self.dfs_l0_query(obs_id=f"{obs_id}") if len(tbl) == 60: - self.logger_ppl.info(f"DFS returns {len(tbl)} records for obs_id={obs_id}") + self.logger_ppl.info( + f"DFS returns {len(tbl)} records for obs_id={obs_id}" + ) else: is_good = False - self.logger_ppl.warning(f"DFS returns {len(tbl)} records for obs_id={obs_id}") + self.logger_ppl.warning( + f"DFS returns {len(tbl)} records for obs_id={obs_id}" + ) return is_good def dfs_l1_query(self, obs_id, detector): - """ Query L1 data from DFS. """ + """Query L1 data from DFS.""" pass def l1_cleanup(self): @@ -1025,20 +1093,28 @@ class CsstMsFile(dict): def fpo(self, post=".fits"): if post.startswith("."): - fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \ - f"{self.obs_id}_{self.detector}_L1_V{self.version}{post}" + fn = ( + f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" + f"{self.obs_id}_{self.detector}_L1_V{self.version}{post}" + ) else: - fn = f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" \ - f"{self.obs_id}_{self.detector}_L1_V{self.version}_{post}" + fn = ( + f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_{self.exp_stop}_" + f"{self.obs_id}_{self.detector}_L1_V{self.version}_{post}" + ) return os.path.join(self.dir_out, fn) def fno(self, post=".fits"): if post.startswith("."): - fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" - f"{self.exp_stop}_{self.obs_id}_{self.detector}_L1_V{self.version}{post}") + fn = ( + f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{self.detector}_L1_V{self.version}{post}" + ) else: - fn = (f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" - f"{self.exp_stop}_{self.obs_id}_{self.detector}_L1_V{self.version}_{post}") + fn = ( + f"{self.telescope}_{self.instrument}_{self.project}_{self.obs_type}_{self.exp_start}_" + f"{self.exp_stop}_{self.obs_id}_{self.detector}_L1_V{self.version}_{post}" + ) return fn def fpi(self): @@ -1059,7 +1135,7 @@ class CsstMsFile(dict): filepath=os.path.join(dfs_root, rec["data"].file_path), ver_sim=ver_sim, dir_out=dir_out, - header=header + header=header, ) def move_to(self, dir_in="./input", dir_out="./output"): @@ -1067,7 +1143,7 @@ class CsstMsFile(dict): filepath=os.path.join(dir_in, os.path.basename(self.filepath)), ver_sim=self.ver_sim, dir_out=dir_out, - **self.kwargs + **self.kwargs, ) def to_dict(self): @@ -1075,11 +1151,13 @@ class CsstMsFile(dict): filepath=self.filepath, ver_sim=self.ver_sim, dir_out=self.dir_out, - kwargs=self.kwargs + kwargs=self.kwargs, ) def __repr__(self): - return f"CsstMsFile(filepath=\"{self.filename}\", ver_sim=\"{self.ver_sim}\", dir_out=\"{self.dir_out}\")" + return f'CsstMsFile(filepath="{self.filename}", ver_sim="{self.ver_sim}", dir_out="{self.dir_out}")' + + # file = CsstMsFile( # "L1/MSC/SCI/62173/10160000108/CSST_MSC_MS_SCI_20290206174352_20290206174622_10160000108_21_L1_V01.fits") # file = CsstMsFile.from_l1id(id=17796, dfs_root="/share/dfs") diff --git a/csst_common/file_recorder.py b/csst_common/file_recorder.py deleted file mode 100644 index 086f995..0000000 --- a/csst_common/file_recorder.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -Identifier: KSC-SJ4-csst_common/__init__.py -Name: __init__.py -Description: csst_common package -Author: Bo Zhang -Created: 2022-09-13 -Modified-History: - 2022-09-13, Bo Zhang, created - 2022-09-13, Bo Zhang, fixed a bug - 2023-12-39, Bo Zhang, deprecated -""" - -import os -from collections import namedtuple - -from astropy import table - -FileRecord = namedtuple("FileRecord", ["filepath", "db", "comment", "existence"]) - - -class FileRecorder(list): - """ - FileRecord Recorder, inherited from the built-in ``list``. - - This recorder is used to record files generated by functional modules. - In principle, a CSST pipeline module should return a status (CsstStatus) - and a file recorder (``FileRecorder``) after it finishes data processing. - - Parameters - ---------- - *args : any - The arguments passed to ``list()``. - **kwargs : any - The keyword arguments passed to ``list()``. - - Methods - ------- - add_record(filepath: str = "", db: bool = False, comment: str = "") - Add a file record, each record is a ``collections.namedtuple``. - Users should provide a file path ``filepath``, whether this file should be - written to database ``db``, a comment ``comment``. - An existence boolean will be attached to check if the file exists. - to_table() - Convert to ``astropy.table.Table``. - pprint(*args, **kwargs) - Use ``astropy.table.Table.pprint`` to print file records in table format. - pprint_all(*args, **kwargs) - Use ``astropy.table.Table.pprint_all`` to print file records in table format. - - Examples - -------- - >>> fr = FileRecorder() - >>> for i in range(3): - >>> fr.add_record("test{:03d}.txt".format(i), db=True, comment="Test file {:d}".format(i)) - >>> fr.pprint_all() - - filepath db comment existence - ----------- ---- ----------- --------- - test000.txt True Test file 0 False - test001.txt True Test file 1 False - test002.txt True Test file 2 False - >>> fr.pprint_all() - - filepath db comment existence - ----------- ---- ----------- --------- - test000.txt True Test file 0 False - test001.txt True Test file 1 False - test002.txt True Test file 2 False - """ - - def __init__(self, *args, **kwargs): - super(FileRecorder, self).__init__(*args, **kwargs) - - @staticmethod - def FileRecord(filepath: str = "", db: bool = False, comment: str = ""): - return FileRecord( - filepath=filepath, - db=db, - comment=comment, - existence=os.path.exists(filepath), - ) - - def add_record(self, filepath: str = "", db: bool = False, comment: str = ""): - existence = os.path.exists(filepath) - assert isinstance(filepath, str) - assert isinstance(db, bool) - assert isinstance(comment, str) - super().append( - FileRecord(filepath=filepath, db=db, comment=comment, existence=existence) - ) - - def to_table(self): - return table.Table([_._asdict() for _ in self]) - - def pprint(self, *args, **kwargs): - print("".format(len(self))) - return self.to_table().pprint(*args, **kwargs) - - def pprint_all(self, *args, **kwargs): - print("".format(len(self))) - return self.to_table().pprint_all(*args, **kwargs) - - def __repr__(self): - t = self.to_table() - lines, outs = t.formatter._pformat_table( - t, - max_lines=-1, - max_width=-1, - show_name=True, - show_unit=None, - show_dtype=False, - align="<", - ) - if outs["show_length"]: - lines.append(f"Length = {len(self)} rows") - return "\n".join(lines) - - @property - def summary(self): - if len(self) == 0: - return "0 alleged, 0 missing, 0 to db" - else: - return ( - f"{len(self)} alleged, " - f"{len(self) - sum(self.to_table()['existence'])} missing, " - f"{sum(self.to_table()['existence'])} to db" - ) - - def is_good(self): - """Check if all alleged files exist.""" - return all(self.to_table()["existence"]) diff --git a/csst_common/io.py b/csst_common/io.py new file mode 100644 index 0000000..e51eaa2 --- /dev/null +++ b/csst_common/io.py @@ -0,0 +1,10 @@ +from astropy.io import fits + + +def check_file(file_path="test.fits") -> bool: + pass + + +def _check_file_fits() -> bool: + """Validate checksum for .fits files.""" + return True diff --git a/csst_common/logger.py b/csst_common/logger.py index ad379e3..a78ac22 100644 --- a/csst_common/logger.py +++ b/csst_common/logger.py @@ -23,9 +23,9 @@ def get_logger(name: str = "CSST pipeline", filename: Optional[str] = None): Parameters ---------- name : str - The logger name. + The logger pipeline_id. filename : str - The log file name. + The log file pipeline_id. Returns ------- @@ -50,7 +50,7 @@ def get_logger(name: str = "CSST pipeline", filename: Optional[str] = None): # logging formatter formatter = logging.Formatter( - "%(asctime)s - %(levelname)s - %(module)s.py:%(lineno)d - %(message)s" + "%(asctime)s - %(levelname)s - %(module)s.py:%(lineno)d - %(msg)s" ) # stream handler diff --git a/csst_common/params.py b/csst_common/params.py index 1dbd85d..d67d59f 100644 --- a/csst_common/params.py +++ b/csst_common/params.py @@ -9,11 +9,14 @@ Modified-History: 2022-09-13, Bo Zhang, added CSST_PARAMS 2022-10-28, Bo Zhang, added DFS_CONF """ +import os.path + import yaml -from . import PACKAGE_PATH + +PACKAGE_PATH = os.path.dirname(__file__) with open(PACKAGE_PATH + "/data/csst_params.yml") as f: CSST_PARAMS = yaml.safe_load(f) -with open(PACKAGE_PATH + "/data/node_conf.yml") as f: - DFS_CONF = yaml.safe_load(f) +# with open(PACKAGE_PATH + "/data/node_conf.yml") as f: +# DFS_CONF = yaml.safe_load(f) diff --git a/csst_common/pipeline.py b/csst_common/pipeline.py index 72ac210..62a6b6e 100644 --- a/csst_common/pipeline.py +++ b/csst_common/pipeline.py @@ -2,94 +2,95 @@ import json import os import subprocess import warnings +from typing import Any from astropy import time -from .crds import CRDS +from .ccds import CCDS from .dfs import DFS from .logger import get_logger class Pipeline: def __init__( - self, - dir_in="/pipeline/input", - dir_out="/pipeline/output", - dir_aux="/pipeline/aux", - dfs_root="/dfsroot", - crds_root="/crdsroot", - crds_cache="/pipeline/crds_cache", - clean=True, - n_jobs_cpu=18, - n_jobs_gpu=9, - device="CPU", - filter_warnings=False, - dfs=True, - crds=False, + self, + dir_input: str = "/pipeline/input", + dir_output: str = "/pipeline/output", + dir_aux: str = "/pipeline/aux", + dfs_root: str = "/dfs_root", + ccds_root: str = "/ccds_root", + ccds_cache: str = "/pipeline/ccds_cache", + filter_warnings: bool = False, + dfs: bool = True, + ccds: bool = False, + **kwargs: Any, ): - # get pipeline information from environ - self.name = os.getenv("PIPELINE", "-") + # get pipeline information from env vars + self.pipeline_id = os.getenv("PIPELINE_ID", "-") self.build = os.getenv("BUILD", "-") + self.created = os.getenv("CREATED", "-") # set directory information - self.dir_in = dir_in - self.dir_out = dir_out + self.dir_input = dir_input + self.dir_output = dir_output self.dir_aux = dir_aux self.dfs_root = dfs_root - self.crds_root = crds_root - self.crds_cache = crds_cache - self.clean = clean + self.ccds_root = ccds_root + self.ccds_cache = ccds_cache - # set resource information - self.n_jobs_cpu = n_jobs_cpu - self.n_jobs_gpu = n_jobs_gpu - self.device = device.upper() + # additional parameters + self.kwargs = kwargs # set logger - self.pipeline_logger = get_logger(name="pipeline", filename=os.path.join(self.dir_out, "pipeline.log")) - self.module_logger = get_logger(name="module", filename=os.path.join(self.dir_out, "module.log")) + self.pipeline_logger = get_logger( + name="pipeline", + filename=os.path.join( + self.dir_output, + "pipeline.log", + ), + ) + self.module_logger = get_logger( + name="module", + filename=os.path.join( + self.dir_output, + "module.log", + ), + ) # change working directory - print(f"Change directory to {self.dir_out}") - os.chdir(self.dir_out) - # clean input/output directory - if self.clean: - self.clean_directory(self.dir_in) - self.clean_directory(self.dir_out) + print(f"Change directory to {self.dir_output}") + os.chdir(self.dir_output) # Frequently used files - self.message = Message(os.path.join(self.dir_out, "msg.txt")) - self.time_stamp = TimeStamp(os.path.join(self.dir_out, "time_stamp.txt")) - self.exit_code = ExitCode(os.path.join(self.dir_out, "exit_code")) - self.error_trace = ErrorTrace(os.path.join(self.dir_out, "error_trace")) + self.msg = MessageWriter( + os.path.join(self.dir_output, "message.txt"), + ) + self.tsr = TimeStampRecorder( + os.path.join(self.dir_output, "time_stamp.txt"), + ) + # self.exit_code = ExitCode(os.path.join(self.dir_output, "exit_code")) + # self.error_trace = ErrorTrace(os.path.join(self.dir_output, "error_trace")) if dfs: self.dfs = DFS(n_try=5) else: self.dfs = None - if crds: - self.crds = CRDS() + if ccds: + self.ccds = CCDS() else: - self.crds = None + self.ccds = None if filter_warnings: self.filter_warnings() - self.ban_multithreading() + def inspect(self): + print(f"PIPELINE_ID={self.pipeline_id}") + print(f"BUILD={self.build}") + print(f"CREATED={self.created}") - def ban_multithreading(self): - os.environ["OMP_NUM_THREADS"] = "1" - os.environ["OPENBLAS_NUM_THREADS"] = "1" - os.environ["MKL_NUM_THREADS"] = "1" - os.environ["VECLIB_MAXIMUM_THREADS"] = "1" - os.environ["NUMEXPR_NUM_THREADS"] = "1" - - def set_test_env(self): - os.environ["CSST_DFS_API_MODE"] = "cluster" - os.environ["CSST_DFS_GATEWAY"] = "172.24.27.2:30880" - os.environ["CSST_DFS_APP_ID"] = "test" - os.environ["CSST_DFS_APP_TOKEN"] = "test" - os.environ["CRDS_SERVER_URL"] = "http://172.24.27.2:29000" + def clean_output(self): + """Clean output directory.""" + self.clean_directory(self.dir_output) @staticmethod def clean_directory(d): @@ -112,19 +113,23 @@ class Pipeline: warnings.resetwarnings() -class ErrorTrace: - def __init__(self, file_path=""): - self.file_path = file_path +# class ErrorTrace: +# """Write error trace to file.""" +# +# def __init__(self, file_path=""): +# self.file_path = file_path +# +# def __repr__(self): +# return f"< ErrorTrace [{self.file_path}] >" +# +# def write(self, s: str): +# with open(self.file_path, "w+") as f: +# f.write(s) - def __repr__(self): - return f"< ErrorTrace [{self.file_path}] >" - def write(self, s: str): - with open(self.file_path, "w+") as f: - f.write(s) +class MessageWriter: + """Write JSON format messages to file.""" - -class Message: def __init__(self, file_path=""): self.file_path = file_path @@ -147,34 +152,47 @@ class Message: return d -class ExitCode: - def __init__(self, file_path=""): - self.file_path = file_path - - def __repr__(self): - return f"< ExitCode [{self.file_path}] >" - - def truncate(self): - with open(self.file_path, 'w') as file: - file.truncate(0) - - def write(self, code=0): - with open(self.file_path, "w+") as f: - f.write(str(code)) - print(f"Exit with code {code} (written to '{self.file_path}')") - - -class TimeStamp: - def __init__(self, file_path=""): +# DEPRECATED +# class ExitCode: +# def __init__(self, file_path=""): +# self.file_path = file_path +# +# def __repr__(self): +# return f"< ExitCode [{self.file_path}] >" +# +# def truncate(self): +# with open(self.file_path, "w") as file: +# file.truncate(0) +# +# def write(self, code=0): +# with open(self.file_path, "w+") as f: +# f.write(str(code)) +# print(f"Exit with code {code} (written to '{self.file_path}')") + + +class TimeStampRecorder: + def __init__(self, file_path: str = "tsr.txt"): + """ + TimeStampRecorder Class. + + Initialize a TimeStampRecorder object anc connect it to `file_path`. + + Parameters + ---------- + file_path : str + Time stamp file path. + """ self.file_path = file_path def __repr__(self): - return f"< TimeStamp [{self.file_path}] >" + return f"< TimeStampRecorder [{self.file_path}] >" - def truncate(self): - with open(self.file_path, 'w') as file: + def empty(self): + """Clean time stamp file.""" + with open(self.file_path, "w") as file: file.truncate(0) - def punch_in(self): + def touch(self): + """Write a time stamp.""" with open(self.file_path, "a+") as f: f.write(f"{time.Time.now().isot}+00:00\n") diff --git a/csst_common/utils/__init__.py b/csst_common/utils/__init__.py index 49c431c..1f3135f 100644 --- a/csst_common/utils/__init__.py +++ b/csst_common/utils/__init__.py @@ -1,3 +1,2 @@ from ._module_docstr import ModuleHeader -from ._io import remove_dir, remove_files from .tempfile import randfile diff --git a/csst_common/utils/_io.py b/csst_common/utils/_io.py deleted file mode 100644 index a459299..0000000 --- a/csst_common/utils/_io.py +++ /dev/null @@ -1,15 +0,0 @@ -import glob -import os -import shutil - - -def remove_files(fmt="*post.fits"): - """ Remove files matching the specified format. """ - for fp in glob.glob(fmt): - os.remove(fp) - - -def remove_dir(path=""): - """ Remove the specified directory. """ - if os.path.exists(path): - shutil.rmtree(path) diff --git a/csst_common/utils/_module_docstr.py b/csst_common/utils/_module_docstr.py index 64c2750..e7c9300 100644 --- a/csst_common/utils/_module_docstr.py +++ b/csst_common/utils/_module_docstr.py @@ -15,7 +15,7 @@ import shutil import numpy as np -from csst_common import PACKAGE_PATH +PACKAGE_PATH = os.path.dirname(os.path.dirname(__file__)) with open(PACKAGE_PATH + "/data/module_header.txt", "r") as f: @@ -50,7 +50,7 @@ class ModuleHeader: identifier: str the identifier, e.g., SJ4 author: str - author name + author pipeline_id description: str description of the module ignore_init: bool diff --git a/csst_common/utils/tempfile.py b/csst_common/utils/tempfile.py index f1aeab9..cfcb493 100644 --- a/csst_common/utils/tempfile.py +++ b/csst_common/utils/tempfile.py @@ -24,4 +24,3 @@ def randfile(digits=20, ext=".fits"): # 使用secrets模块生成指定长度的随机字符串 random_string = ''.join(secrets.choice(characters) for _ in range(digits)) + ext return random_string - diff --git a/csst_common/wrapper.py b/csst_common/wrapper.py deleted file mode 100644 index a8ddb81..0000000 --- a/csst_common/wrapper.py +++ /dev/null @@ -1,113 +0,0 @@ -import functools -import time -import traceback -from collections import namedtuple - -from astropy.time import Time - -from csst_common.data_manager import CsstMsDataManager -from csst_common.file_recorder import FileRecorder -from csst_common.status import CsstStatus - -__all__ = ["ModuleResult", "l1ppl_module", "module_wrapper"] - -# module should return ModuleResult as result -ModuleResult = namedtuple("ModuleResult", ["module", "timecost", "status", "fr", "output"]) - - -def l1ppl_module(func): - @functools.wraps(func) - def call_l1ppl_module(dm: CsstMsDataManager, *args, **kwargs): - dm.logger_ppl.info(f"=====================================================") - t_start = time.time() - dm.logger_ppl.info(f"Starting Module: **{func.__name__}**") - # dm.logger_ppl.info(f"Additional arguments: {args} {kwargs}") - try: - # if the module finishes - status, fr, *output = func(dm, *args, **kwargs) - except Exception as e: - # if the module raises error - exc_info = traceback.format_exc() # traceback info - dm.logger_ppl.error(f"Error occurs! \n{exc_info}") - status = CsstStatus.ERROR # default status if exceptions occur - fr = FileRecorder() # default FileRecorder if exceptions occur - output = [exc_info, ] # default output if exceptions occur - finally: - t_stop = time.time() - t_cost = t_stop - t_start - if status in [CsstStatus.PERFECT, CsstStatus.WARNING, CsstStatus.ERROR]: - # perfect / warning / error - dm.logger_ppl.info(f"Module finished with status: {status} - time cost: {t_cost:.1f} sec") - else: - # invalid status - dm.logger_ppl.error(f"Invalid status: {status}") - # record exception traceback info - dm.logger_ppl.info( - f"ModuleResult: \n" - f" - name: {func.__name__}\n" - f" - status: {status}\n" - f" - additional output: {output}\n" - f" - fr: [{fr.summary}]\n{fr}\n" - ) - # write time stamp - dm.write_stamp() - return ModuleResult(func.__name__, t_cost, status, fr, output) - - return call_l1ppl_module - - -def module_wrapper(func): - @functools.wraps(func) - def call_module(logger, tstamp=None, *args, **kwargs): - logger.info(f"=====================================================") - t_start = time.time() - logger.info(f"Starting Module: **{func.__name__}**") - # logger.info(f"Additional arguments: {args} {kwargs}") - try: - # if the module finishes - status, fr, *output = func(*args, **kwargs) - except Exception as e: - # if the module raises error - exc_info = traceback.format_exc() # traceback info - logger.error(f"Error occurs! \n{exc_info}") - status = CsstStatus.ERROR # default status if exceptions occur - fr = FileRecorder() # default FileRecorder if exceptions occur - output = [exc_info, ] # default output if exceptions occur - finally: - t_stop = time.time() - t_cost = t_stop - t_start - if status in [CsstStatus.PERFECT, CsstStatus.WARNING, CsstStatus.ERROR]: - # perfect / warning / error - logger.info(f"Module finished with status: {status} - time cost: {t_cost:.1f} sec") - else: - # invalid status - logger.error(f"Invalid status: {status}") - # record exception traceback info - logger.info( - f"ModuleResult: \n" - f" - name: {func.__name__}\n" - f" - status: {status}\n" - f" - additional output: {output}\n" - f" - fr: [{fr.summary}]\n{fr}\n" - ) - # write time stamp - if tstamp is not None: - with open(tstamp, "a+") as f: - f.write(f"{Time.now().isot}+08:00\n") - return ModuleResult(func.__name__, t_cost, status, fr, output) - - return call_module - - -if __name__ == "__main__": - - @l1ppl_module - def call_add(dm, a, b): - if isinstance(a, float) and isinstance(b, float): - return ModuleResult(CsstStatus.PERFECT, None, a + b) - else: - return ModuleResult(CsstStatus.ERROR, None, (a, b)) - - # dm = CsstMsDataManager() - # print(call_add(dm, 1., 2.)) - # print(call_add(dm, 1., None)) diff --git a/tests/test_data_manager.py b/tests/test_data_manager.py index 7234473..82114ce 100644 --- a/tests/test_data_manager.py +++ b/tests/test_data_manager.py @@ -1,50 +1,50 @@ -""" -Identifier: KSC-SJ4-tests/test_data_manager.py -Name: test_data_manager.py -Description: data manager unit test -Author: Bo Zhang -Created: 2022-09-13 -Modified-History: - 2022-09-13, Bo Zhang, created - 2022-09-29, Bo Zhang, added test for CsstMbiDataManager - 2022-10-28, Bo Zhang, deleted unit test for CsstMsDataManager -""" -import os -import unittest - -from csst_common.data_manager import CsstMsDataManager -from csst_common.params import CSST_PARAMS as CP - -dir_unittest = "/nfsdata/share/pipeline-unittest/csst_common" - - -class TestCsstMsDataManager(unittest.TestCase): - def setUp(self) -> None: - self.dm_mbi = CsstMsDataManager.quickstart( - ver_sim="C6.2", datatype="mbi", dir_l1=dir_unittest, exposure_id=100) - self.dm_sls = CsstMsDataManager.quickstart( - ver_sim="C6.2", datatype="sls", dir_l1=dir_unittest, exposure_id=100) - self.dm_mbi.target_detectors = None - self.dm_sls.target_detectors = None - - def test_mbi_data_existence(self): - self.assertTrue(self.dm_mbi.target_detectors == CP["mbi"]["detectors"]) - self.assertTrue(os.path.exists(self.dm_mbi.l0_detector(6))) - self.assertTrue(os.path.exists(self.dm_mbi.l0_log(6))) - self.assertTrue(os.path.exists(self.dm_mbi.l0_cat(6))) - self.assertTrue(os.path.exists(self.dm_mbi.l0_crs(6))) - self.assertTrue(isinstance(self.dm_mbi.l1_detector(6, post="img.fits"), str)) - self.assertTrue(isinstance(self.dm_mbi.l1_file(name="some_file.ext", comment="a demo file"), str)) - - def test_sls_data_existence(self): - self.assertTrue(self.dm_sls.target_detectors == CP["sls"]["detectors"]) - self.assertTrue(os.path.exists(self.dm_sls.l0_detector(1))) - self.assertTrue(os.path.exists(self.dm_sls.l0_log(1))) - self.assertTrue(os.path.exists(self.dm_sls.l0_cat(1))) - self.assertTrue(os.path.exists(self.dm_sls.l0_crs(1))) - self.assertTrue(isinstance(self.dm_sls.l1_detector(1, post="flt.fits"), str)) - self.assertTrue(isinstance(self.dm_sls.l1_file(name="some_file.ext", comment="a demo file"), str)) - - # DFS is not always available - # def test_dfs_is_available(self): - # self.assertTrue(self.dm_mbi.dfs_is_available()) +# """ +# Identifier: KSC-SJ4-tests/test_data_manager.py +# Name: test_data_manager.py +# Description: data manager unit test +# Author: Bo Zhang +# Created: 2022-09-13 +# Modified-History: +# 2022-09-13, Bo Zhang, created +# 2022-09-29, Bo Zhang, added test for CsstMbiDataManager +# 2022-10-28, Bo Zhang, deleted unit test for CsstMsDataManager +# """ +# import os +# import unittest +# +# from csst_common.data_manager import CsstMsDataManager +# from csst_common.params import CSST_PARAMS as CP +# +# dir_unittest = "/nfsdata/share/pipeline-unittest/csst_common" +# +# +# class TestCsstMsDataManager(unittest.TestCase): +# def setUp(self) -> None: +# self.dm_mbi = CsstMsDataManager.quickstart( +# ver_sim="C6.2", datatype="mbi", dir_l1=dir_unittest, exposure_id=100) +# self.dm_sls = CsstMsDataManager.quickstart( +# ver_sim="C6.2", datatype="sls", dir_l1=dir_unittest, exposure_id=100) +# self.dm_mbi.target_detectors = None +# self.dm_sls.target_detectors = None +# +# def test_mbi_data_existence(self): +# self.assertTrue(self.dm_mbi.target_detectors == CP["mbi"]["detectors"]) +# self.assertTrue(os.path.exists(self.dm_mbi.l0_detector(6))) +# self.assertTrue(os.path.exists(self.dm_mbi.l0_log(6))) +# self.assertTrue(os.path.exists(self.dm_mbi.l0_cat(6))) +# self.assertTrue(os.path.exists(self.dm_mbi.l0_crs(6))) +# self.assertTrue(isinstance(self.dm_mbi.l1_detector(6, post="img.fits"), str)) +# self.assertTrue(isinstance(self.dm_mbi.l1_file(pipeline_id="some_file.ext", comment="a demo file"), str)) +# +# def test_sls_data_existence(self): +# self.assertTrue(self.dm_sls.target_detectors == CP["sls"]["detectors"]) +# self.assertTrue(os.path.exists(self.dm_sls.l0_detector(1))) +# self.assertTrue(os.path.exists(self.dm_sls.l0_log(1))) +# self.assertTrue(os.path.exists(self.dm_sls.l0_cat(1))) +# self.assertTrue(os.path.exists(self.dm_sls.l0_crs(1))) +# self.assertTrue(isinstance(self.dm_sls.l1_detector(1, post="flt.fits"), str)) +# self.assertTrue(isinstance(self.dm_sls.l1_file(pipeline_id="some_file.ext", comment="a demo file"), str)) +# +# # DFS is not always available +# # def test_dfs_is_available(self): +# # self.assertTrue(self.dm_mbi.dfs_is_available()) diff --git a/tests/test_file_recorder.py b/tests/test_file_recorder.py deleted file mode 100644 index d2c43d0..0000000 --- a/tests/test_file_recorder.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Identifier: KSC-SJ4-tests/test_data_manager.py -Name: test_data_manager.py -Description: data manager unit test -Author: Bo Zhang -Created: 2022-09-13 -Modified-History: - 2022-10-10, Bo Zhang, created - 2022-10-10, Bo Zhang, added test for FileRecorder -""" -import unittest -from csst_common.file_recorder import FileRecorder, FileRecord - - -# deprecated in some days -class TestCsstMbiDataManager(unittest.TestCase): - def setUp(self) -> None: - self.fr = FileRecorder() - - def test_add_record(self): - self.fr.add_record(filepath="test1", db=False, comment="the first test") - self.assertTrue(self.fr[0] == FileRecord("test1", db=False, comment="the first test", existence=False)) diff --git a/tests/test_params.py b/tests/test_params.py deleted file mode 100644 index 8387e15..0000000 --- a/tests/test_params.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Identifier: KSC-SJ4-tests/test_params.py -Name: test_params.py -Description: unit test for CSST parameters -Author: Bo Zhang -Created: 2022-09-13 -Modified-History: - 2022-09-13, Bo Zhang, created - 2022-09-13, Bo Zhang, fixed a bug -""" -import unittest - - -class TestParams(unittest.TestCase): - def test_params(self): - self.assertTrue(True) -- GitLab