Commit b026df97 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

major updates

parent cad9fd06

.gitignore

+1 −0
Original line number Original line Diff line number Diff line 
unit_test_data_root

.idea/*

.idea/*

*.png

*.png

*.DS_Store*

*.DS_Store*

Makefile

+1 −1
Original line number Original line Diff line number Diff line
@@ -12,6 +12,6 @@ uninstall: 
	pip uninstall $(PKG) -y

	pip uninstall $(PKG) -y





test:

test:

	coverage run -m  pytest . --import-mode=importlib --cov-report=html --cov-report=term-missing

	coverage run -m  pytest . --cov=$(PKG) --import-mode=importlib --cov-report=html --cov-report=term-missing

	coverage report -m

	coverage report -m

	rm -rf .coverage .pytest_cache
 
	rm -rf .coverage .pytest_cache

README.md

+0 −39
Original line number Original line Diff line number Diff line
@@ -44,42 +44,3 @@ from csst_common.params import CSST_PARAMS as CP 
print(CP)

print(CP)

print(CP["mbi"]["detector2filter"])

print(CP["mbi"]["detector2filter"])

```
 
```
 


how to use data_manager

```python

from csst_common.data_manager import CsstMbiDataManager

dm = CsstMbiDataManager(...)

# access L0 directory

dm.dir_l0

# access L1 directory

dm.dir_l1

# access dir_pcref

dm.dir_pcref

# access path_aux

dm.path_aux

# access ver_sim

dm.ver_sim

# access target detectors

dm.target_detectors

# access available detectors

dm.available_detectors

# define an L1 file (detector-specified)

dm.l1_detector(detector=6)

# define an L1 file (non-detector-specified)

dm.l1_file("flipped_image.fits")

```



a shortcut for test dataset



```python

from csst_common.data_manager import CsstMbiDataManager



CsstMbiDataManager.quickstart(ver_sim="C5.2", dir_l1=".", exposure_id=100)

```



## algorithm description



The `csst_common` provides some common modules for CSST pipeline.



- `csst_common.parameters`

- `csst_common.data_manager`

csst_common/__init__.py

+5 −5
Original line number Original line Diff line number Diff line
@@ -9,10 +9,10 @@ Modified-History: 
    2022-09-13, Bo Zhang, fixed a bug

    2022-09-13, Bo Zhang, fixed a bug

"""

"""





import os



from .status import CsstResult, CsstStatus

from .status import CsstResult, CsstStatus

from .pipeline import Pipeline

from .ccds import CCDS

from .dfs import DFS

from .decorator import parameterized_module_decorator





__version__ = "0.0.1"

__version__ = "0.0.2"
 


PACKAGE_PATH = os.path.dirname(__file__)

csst_common/ccds/slsconf.py

0 → 100644
+350 −0
Original line number Original line Diff line number Diff line 
import json

import os

import re



import astropy.io.fits as pyfits



from ccds import client





# chipid: [01,   02,   03,   04,   05,   10,   21,   26,   27,   28,   29,   30]

# filter: [GI,   GV,   GU,   GU,   GV,   GI,   GI,   GV,   GU,   GU,   GV,   GI]

# clabel: [GI-1, GV-1, GU-1, GU-2, GV-2, GI-2, GI-3, GV-3, GU-3, GU-4, GV-4, GI-4]





def get_version():

    observatory = client.get_default_observatory()

    operational_context = client.get_default_context(observatory)

    ver = re.split(r"[_.]", operational_context)[1]

    return ver





def resave_sensitivity(dir_save, chipid, flt):

    ver = get_version()

    h = pyfits.open(

        dir_save + "CSST_MSC_MS_SENSITIVITY_" + chipid + "_" + ver + ".fits"

    )

    for extname, order in zip(

        ["L0ST", "LP1ST", "LM1ST", "LP2ST", "LM2ST"],

        ["0st", "1st", "-1st", "2st", "-2st"],

    ):

        hdu0 = pyfits.PrimaryHDU()

        hdu1 = pyfits.BinTableHDU(h[extname].data)

        hdul = pyfits.HDUList([hdu0, hdu1])

        hdul.writeto(

            dir_save + "CSST_MSC_MS_SENSITIVITY_" + flt + "_" + order + ".fits",

            overwrite=True,

            checksum=True,

        )





def readjson(dir_save, fjsoname):

    with open(dir_save + fjsoname) as f:

        d = json.load(f)

    return d





##################################################################################

##################################################################################



# keys = ['BEAMA', 'MMAG_EXTRACT_A', 'MMAG_MARK_A', 'DYDX_ORDER_A', 'DYDX_A_0', 'DYDX_A_1', 'XOFF_A', 'YOFF_A',

#         'DISP_ORDER_A', 'DLDP_A_0', 'DLDP_A_1', 'BEAMB', 'MMAG_EXTRACT_B', 'MMAG_MARK_B', 'DYDX_ORDER_B', 'DYDX_B_0',

#         'XOFF_B', 'YOFF_B', 'DISP_ORDER_B', 'DLDP_B_0', 'DLDP_B_1', 'BEAMC', 'MMAG_EXTRACT_C', 'MMAG_MARK_C',

#         'DYDX_ORDER_C', 'DYDX_C_0', 'DYDX_C_1', 'XOFF_C', 'YOFF_C', 'DISP_ORDER_C', 'DLDP_C_0', 'DLDP_C_1', 'BEAMD',

#         'MMAG_EXTRACT_D', 'MMAG_MARK_D', 'DYDX_ORDER_D', 'DYDX_D_0', 'DYDX_D_1', 'XOFF_D', 'YOFF_D', 'DISP_ORDER_D',

#         'DLDP_D_0', 'DLDP_D_1', 'BEAME', 'MMAG_EXTRACT_E', 'MMAG_MARK_E', 'DYDX_ORDER_E', 'DYDX_E_0', 'DYDX_E_1',

#         'XOFF_E', 'YOFF_E', 'DISP_ORDER_E', 'DLDP_E_0', 'DLDP_E_1']



GL = [

    "GI2",

    "GV4",

    "GU2",

    "GU4",

    "GV2",

    "GI4",

    "GI6",

    "GV8",

    "GU6",

    "GU8",

    "GV6",

    "GI8",

]

GR = [

    "GI1",

    "GV3",

    "GU1",

    "GU3",

    "GV1",

    "GI3",

    "GI5",

    "GV7",

    "GU5",

    "GU7",

    "GV5",

    "GI7",

]

SEN = ["GI", "GV", "GU", "GU", "GV", "GI", "GI", "GV", "GU", "GU", "GV", "GI"]





def fwriteKEY(fsx, i):

    fsx.write("INSTRUMENT CSSTSLS" + "\n")

    fsx.write("CAMERA " + SEN[i] + "\n")

    if SEN[i] == "GI":

        fsx.write("WAVELENGTH 6200 10000" + "\n")

    elif SEN[i] == "GV":

        fsx.write("WAVELENGTH 4000 6200" + "\n")

    elif SEN[i] == "GU":

        fsx.write("WAVELENGTH 2550 4000" + "\n")

    fsx.write("\n" + "SCIENCE_EXT SCI ; Science extension" + "\n")

    fsx.write("DQ_EXT DQ       ; DQ extension" + "\n")

    fsx.write("ERRORS_EXT ERR  ; Error extension" + "\n")

    fsx.write("FFNAME csstFlat.fits" + "\n")

    fsx.write("DQMASK 246    ; 4096 and 512 taken out" + "\n")

    fsx.write("\n" + "RDNOISE 5.0" + "\n")

    fsx.write("EXPTIME EXPTIME" + "\n")

    fsx.write("POBJSIZE 1.0" + "\n")

    fsx.write("#SMFACTOR 1.0" + "\n\n")





def fwriteBEAM(

    dir_save,

    fsx,

    i,

    GRATINGLR,

    BEAMX,

    MMAG_EXTRACT_X,

    MMAG_MARK_X,

    DYDX_ORDER_X,

    DYDX_X_0,

    DYDX_X_1,

    XOFF_X,

    YOFF_X,

    DISP_ORDER_X,

    DLDP_X_0,

    DLDP_X_1,

):

    ver = get_version()

    [

        d01_GI21,

        d02_GV43,

        d03_GU21,

        d04_GU43,

        d05_GV21,

        d10_GI43,

        d21_GI65,

        d26_GV87,

        d27_GU65,

        d28_GU87,

        d29_GV65,

        d30_GI87,

    ] = [

        readjson(dir_save, "CSST_MSC_MS_EXTRACT1D_" + chipid + "_" + ver + ".json")

        for chipid in [

            "01",

            "02",

            "03",

            "04",

            "05",

            "10",

            "21",

            "26",

            "27",

            "28",

            "29",

            "30",

        ]

    ]

    d = [

        d01_GI21,

        d02_GV43,

        d03_GU21,

        d04_GU43,

        d05_GV21,

        d10_GI43,

        d21_GI65,

        d26_GV87,

        d27_GU65,

        d28_GU87,

        d29_GV65,

        d30_GI87,

    ]

    fsx.write(BEAMX), [

        fsx.write(" " + str(d[i][GRATINGLR][BEAMX][j]))

        for j in range(len(d[i][GRATINGLR][BEAMX]))

    ], fsx.write("\n")

    fsx.write(MMAG_EXTRACT_X + " " + str(d[i][GRATINGLR][MMAG_EXTRACT_X]) + "\n")

    fsx.write(MMAG_MARK_X + " " + str(d[i][GRATINGLR][MMAG_MARK_X]) + "\n")

    fsx.write("# " + "\n")

    fsx.write("# Trace description " + "\n")

    fsx.write("# " + "\n")

    fsx.write(DYDX_ORDER_X + " " + str(d[i][GRATINGLR][DYDX_ORDER_X]) + "\n")

    fsx.write(DYDX_X_0), [

        fsx.write(" " + str(d[i][GRATINGLR][DYDX_X_0][j]))

        for j in range(len(d[i][GRATINGLR][DYDX_X_0]))

    ], fsx.write("\n")

    if BEAMX == "BEAMB":

        pass

    else:

        fsx.write(DYDX_X_1), [

            fsx.write(" " + str(d[i][GRATINGLR][DYDX_X_1][j]))

            for j in range(len(d[i][GRATINGLR][DYDX_X_1]))

        ], fsx.write("\n")

    fsx.write("# " + "\n")

    fsx.write("# X and Y Offsets " + "\n")

    fsx.write("# " + "\n")

    fsx.write(XOFF_X + " " + str(d[i][GRATINGLR][XOFF_X]) + "\n")

    fsx.write(YOFF_X + " " + str(d[i][GRATINGLR][YOFF_X]) + "\n")

    fsx.write("# " + "\n")

    fsx.write("# Dispersion solution " + "\n")

    fsx.write("# " + "\n")

    fsx.write(DISP_ORDER_X + " " + str(d[i][GRATINGLR][DISP_ORDER_X]) + "\n")

    fsx.write(DLDP_X_0), [

        fsx.write(" " + str(d[i][GRATINGLR][DLDP_X_0][j]))

        for j in range(len(d[i][GRATINGLR][DLDP_X_0]))

    ], fsx.write("\n")

    fsx.write(DLDP_X_1), [

        fsx.write(" " + str(d[i][GRATINGLR][DLDP_X_1][j]))

        for j in range(len(d[i][GRATINGLR][DLDP_X_1]))

    ], fsx.write("\n")

    fsx.write("# " + "\n")

    if BEAMX == "BEAMA":

        ordername = "1st"

        fsx.write(

            "SENSITIVITY_A CSST_MSC_MS_SENSITIVITY_" + SEN[i] + "_1st.fits" + "\n"

        )

    elif BEAMX == "BEAMB":

        ordername = "0st"

        fsx.write(

            "SENSITIVITY_B CSST_MSC_MS_SENSITIVITY_" + SEN[i] + "_0st.fits" + "\n"

        )

    elif BEAMX == "BEAMC":

        ordername = "-1st"

        fsx.write(

            "SENSITIVITY_C CSST_MSC_MS_SENSITIVITY_" + SEN[i] + "_-1st.fits" + "\n"

        )

    elif BEAMX == "BEAMD":

        ordername = "2st"

        fsx.write(

            "SENSITIVITY_D CSST_MSC_MS_SENSITIVITY_" + SEN[i] + "_2st.fits" + "\n"

        )

    elif BEAMX == "BEAME":

        ordername = "-2st"

        fsx.write(

            "SENSITIVITY_E CSST_MSC_MS_SENSITIVITY_" + SEN[i] + "_-2st.fits" + "\n"

        )

    fsx.write("# " + "\n" + "\n")





def fsave_conf(dir_save, GLR, GRATINGLR, i):

    c = dir_save + "CSST_MSC_MS_" + GLR[i] + ".conf"

    os.system("> " + c)

    fs = open(c, "a")

    fwriteKEY(fs, i)

    fs.write("# 1 order (BEAM A) *******************" + "\n")

    fwriteBEAM(

        dir_save,

        fs,

        i,

        GRATINGLR,

        "BEAMA",

        "MMAG_EXTRACT_A",

        "MMAG_MARK_A",

        "DYDX_ORDER_A",

        "DYDX_A_0",

        "DYDX_A_1",

        "XOFF_A",

        "YOFF_A",

        "DISP_ORDER_A",

        "DLDP_A_0",

        "DLDP_A_1",

    )

    fs.write("\n# 0 order (BEAM B) *******************" + "\n")

    fwriteBEAM(

        dir_save,

        fs,

        i,

        GRATINGLR,

        "BEAMB",

        "MMAG_EXTRACT_B",

        "MMAG_MARK_B",

        "DYDX_ORDER_B",

        "DYDX_B_0",

        "DYDX_B_1",

        "XOFF_B",

        "YOFF_B",

        "DISP_ORDER_B",

        "DLDP_B_0",

        "DLDP_B_1",

    )

    fs.write("\n# -1 order (BEAM C) *******************" + "\n")

    fwriteBEAM(

        dir_save,

        fs,

        i,

        GRATINGLR,

        "BEAMC",

        "MMAG_EXTRACT_C",

        "MMAG_MARK_C",

        "DYDX_ORDER_C",

        "DYDX_C_0",

        "DYDX_C_1",

        "XOFF_C",

        "YOFF_C",

        "DISP_ORDER_C",

        "DLDP_C_0",

        "DLDP_C_1",

    )

    fs.write("\n# 2 order (BEAM D) *******************" + "\n")

    fwriteBEAM(

        dir_save,

        fs,

        i,

        GRATINGLR,

        "BEAMD",

        "MMAG_EXTRACT_D",

        "MMAG_MARK_D",

        "DYDX_ORDER_D",

        "DYDX_D_0",

        "DYDX_D_1",

        "XOFF_D",

        "YOFF_D",

        "DISP_ORDER_D",

        "DLDP_D_0",

        "DLDP_D_1",

    )

    fs.write("\n# -2 order (BEAM E) *******************" + "\n")

    fwriteBEAM(

        dir_save,

        fs,

        i,

        GRATINGLR,

        "BEAME",

        "MMAG_EXTRACT_E",

        "MMAG_MARK_E",

        "DYDX_ORDER_E",

        "DYDX_E_0",

        "DYDX_E_1",

        "XOFF_E",

        "YOFF_E",

        "DISP_ORDER_E",

        "DLDP_E_0",

        "DLDP_E_1",

    )

    fs.close()





def get_slsconf(dir_save=".", **kwargs):

    """save SLS conf files to `dir_save`"""



    # resave the sensitivity.fits

    for chipid, flt in zip(["01", "02", "03"], ["GI", "GV", "GU"]):

        resave_sensitivity(dir_save, chipid, flt)



    # save CSST_MSC_MS_*.conf

    for i in range(0, 12):

        fsave_conf(dir_save, GL, "GRATINGL", i)

        fsave_conf(dir_save, GR, "GRATINGR", i)



    # TODO: assert all files are saved correctly, then return



    # TODO: return a dict containing filepath mapping