Commit 8c6e9c2c authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

finished pipeline for instrumental correction

parent 5d32ace3

csst/__init__.py

+1 −0
Original line number Diff line number Diff line 
import os



__version__ = "0.0.5"



PACKAGE_PATH = os.path.dirname(__file__)

csst/msc/astrometry.py

+8 −1
Original line number Diff line number Diff line 
from ..core.processor import CsstProcessor

from .. import PACKAGE_PATH



CONFIG_SCAMP = PACKAGE_PATH + "/msc/config/test.txt"





class CsstProcMscPositionCalibration(CsstProcessor):
@@ -10,9 +13,13 @@ class CsstProcMscPositionCalibration(CsstProcessor): 


    def run(self, data, *args, **kwargs):

        # run your pipeline here

        # make sure that your input data should be a child class instance of CsstData.

        # make sure that your input raw should be a child class instance of CsstData.

        pass



    def cleanup(self, **kwargs):

        # clean up environment

        pass





def test_dir():

    print(CONFIG_SCAMP)
 
 No newline at end of file

csst/msc/instrument.py

+6 −2
Original line number Diff line number Diff line
@@ -94,7 +94,7 @@ class CsstMscInstrumentProc(CsstProcessor): 
            inpaint_model = 'ACS-WFC-F606W-2-32'

            model = deepCR(clean_model, inpaint_model, device='CPU', hidden=50)

            masked, cleaned = model.clean(

                self.__img, threshold=0.5, inpaint=True, segment=True, patch=256, parallel=True, n_jobs=2)

                self.__img, threshold=0.5, inpaint=True, segment=True, patch=256, parallel=True, n_jobs=self.n_jobs)

        else:

            cleaned, masked = cosmicray_lacosmic(ccd=self.__img,

                                                 sigclip=3.,  # cr_threshold
@@ -137,7 +137,8 @@ class CsstMscInstrumentProc(CsstProcessor): 
        weight[self.__flg > 0] = 0

        self.__wht = weight



    def prepare(self, **kwargs):

    def prepare(self, n_jobs=2, **kwargs):

        self.n_jobs = n_jobs

        for name in kwargs:

            self._switches[name] = kwargs[name]
@@ -165,6 +166,9 @@ class CsstMscInstrumentProc(CsstProcessor): 
        img = raw.deepcopy(name="SCI", data=self.__img)

        wht = raw.deepcopy(name="WHT", data=self.__wht)

        flg = raw.deepcopy(name="FLG", data=self.__flg)

        img.set_keyword("FILENAME", img.get_keyword("FILENAME", hdu=0).replace("_raw", "_img"), hdu=0)

        wht.set_keyword("FILENAME", wht.get_keyword("FILENAME", hdu=0).replace("_raw", "_img"), hdu=0)

        flg.set_keyword("FILENAME", flg.get_keyword("FILENAME", hdu=0).replace("_raw", "_img"), hdu=0)



        return img, wht, flg

csst/msc/pipeline.py

+26 −60
Original line number Diff line number Diff line
@@ -4,82 +4,48 @@ 
# node name

HOST = "tulip"

# working directory

DIR_WORKING = ""

DIR_WORK = "/share/HDD7/csstpipeline/msc"

# gaia catalog directory (for position calibration)

DIR_GAIA_CATALOG = ""



DIR_TEST = "/share/Cycle-3-SimuData/multipleBandsImaging/CSST_shearOFF/MSC_0000020/"  #  MSC_MS_210525220000_100000020_06_raw.fits

PATH_BIAS = "/share/HDD7/csstpipeline/ref/MSC_CLB_210525200000_100000016_{:02d}_combine.fits"

PATH_DARK = "/share/HDD7/csstpipeline/ref/MSC_CLD_210525202000_100000016_{:02d}_combine.fits"

PATH_FLAT = "/share/HDD7/csstpipeline/ref/MSC_CLF_210525201000_100000016_{:02d}_combine.fits"



# raw images

fp_list = ["MSC_MS_210527171000_100000279_6_raw.fits",

           "MSC_MS_210527171000_100000279_7_raw.fits",

           "MSC_MS_210527171000_100000279_8_raw.fits",

           "MSC_MS_210527171000_100000279_9_raw.fits",

           "MSC_MS_210527171000_100000279_10_raw.fits",

           "MSC_MS_210527171000_100000279_11_raw.fits",

           "MSC_MS_210527171000_100000279_12_raw.fits",

           "MSC_MS_210527171000_100000279_13_raw.fits",

           "MSC_MS_210527171000_100000279_14_raw.fits",

           "MSC_MS_210527171000_100000279_15_raw.fits",

           "MSC_MS_210527171000_100000279_16_raw.fits",

           "MSC_MS_210527171000_100000279_17_raw.fits",

           "MSC_MS_210527171000_100000279_18_raw.fits",

           "MSC_MS_210527171000_100000279_19_raw.fits",

           "MSC_MS_210527171000_100000279_20_raw.fits",

           "MSC_MS_210527171000_100000279_21_raw.fits",

           "MSC_MS_210527171000_100000279_22_raw.fits",

           "MSC_MS_210527171000_100000279_23_raw.fits",

           "MSC_MS_210527171000_100000279_24_raw.fits",

           "MSC_MS_210527171000_100000279_25_raw.fits",

           ]

import os

os.chdir(DIR_WORK)



import glob

from csst.msc.data import CsstMscImgData

from csst.msc.instrument import CsstMscInstrumentProc

from astropy.io import fits

# from astropy.io import fits



# get aux raw

bs = fits.getdata("/raw/ref/MSC_CLB_210525190000_100000014_13_combine.fits")

dk = fits.getdata("/raw/ref/MSC_CLD_210525192000_100000014_13_combine.fits")

ft = fits.getdata("/raw/ref/MSC_CLF_210525191000_100000014_13_combine.fits")

for i_ccd in range(6, 26):

    # i_ccd = 6

    print("processing CCD {}".format(i_ccd))

    fp_raw = glob.glob("{}/MSC_MS_*{:02}_raw.fits".format(DIR_TEST, i_ccd))

    assert len(fp_raw) == 1

    fp_raw = fp_raw[0]



fp_img_list = []

fp_flg_list = []

fp_wht_list = []

data_list = []



for fp in fp_list:

    # read image raw

    data = CsstMscImgData.read(fp)



    # set aux raw

    data.set_bias(bs)

    data.set_dark(dk)

    data.set_flat(ft)

    raw = CsstMscImgData.read(fp_raw)

    bias = raw.get_bias(PATH_BIAS.format(i_ccd))

    dark = raw.get_bias(PATH_DARK.format(i_ccd))

    flat = raw.get_bias(PATH_FLAT.format(i_ccd))



    # initialize Instrument Processor

    instProc = CsstMscInstrumentProc()

    instProc.prepare()

    instProc.run(data)

    instProc.prepare(n_jobs=2)

    img, wht, flg = instProc.run(raw, bias, dark, flat)

    instProc.cleanup()



    # output filepath

    fp_img = fp.replace("raw.fits", "img.fits")

    fp_flg = fp.replace("raw.fits", "flg.fits")

    fp_wht = fp.replace("raw.fits", "wht.fits")



    # save l1 raw

    data.save_l1data('sci', fp_img)

    data.save_l1data('flag', fp_flg)

    data.save_l1data('weight', fp_wht)



    # store in a list for position calibration

    fp_img_list.append(fp_img)

    fp_flg_list.append(fp_flg)

    fp_wht_list.append(fp_wht)



    data_list.append(data)

    # save img, wht, flg to somewhere

    img.writeto("{}/{}.fits".format(DIR_WORK, img.get_keyword("FILENAME")))

    wht.writeto("{}/{}.fits".format(DIR_WORK, wht.get_keyword("FILENAME")))

    flg.writeto("{}/{}.fits".format(DIR_WORK, flg.get_keyword("FILENAME")))





# position calibration

# TODO:  position calibration

from csst.msc.astrometry import CsstProcMscPositionCalibration

pcProc = CsstProcMscPositionCalibration()

pcProc.prepare(search_radius=2.,)