Commit ba060b49 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

updated pipeline

parent 732d8334

csst/msc/phot.py

+6 −5
Original line number Original line Diff line number Diff line 
from pathlib import Path

from pathlib import Path



import joblib

import numpy as np

import numpy as np

from ccdproc import cosmicray_lacosmic

from ccdproc import cosmicray_lacosmic

from deepCR import deepCR

from deepCR import deepCR
@@ -13,19 +13,20 @@ from .. import PACKAGE_PATH 
# DEEPCR_MODEL_PATH = PACKAGE_PATH + "/msc/deepcr_model/CSST_2021-12-30_CCD23_epoch20.pth"

# DEEPCR_MODEL_PATH = PACKAGE_PATH + "/msc/deepcr_model/CSST_2021-12-30_CCD23_epoch20.pth"









class CsstMscInPhotometryProc(CsstProcessor):

class CsstMscPhotometryProc(CsstProcessor):





    def __init__(self):

    def __init__(self):

        super(CsstProcessor, self).__init__()

        super(CsstProcessor, self).__init__()





    def prepare(self, **kwargs):

    def prepare(self, **kwargs):

        # check whether SExtractor and PSFEx exists

        pass

        pass





    def cleanup(self):

    def cleanup(self):

        pass

        pass





    def run(self, fl_in, out_dir):

    def run(self, fl_in, out_dir, n_jobs=1, verbose=5):

        # assert len(fl_in) == len(fl_out)

        # assert len(fl_in) == len(fl_out)

        for i in range(len(fl_in)):

        joblib.Parallel(n_jobs=n_jobs, verbose=verbose)(

            do_phot(fl_in[i], outdir=out_dir)

            joblib.delayed(do_phot)(fl_in[i], outdir=out_dir) for i in range(len(fl_in)))

        return
 
        return

csst/msc/pipeline.py

+16 −6
Original line number Original line Diff line number Diff line
@@ -3,8 +3,8 @@ 
import glob

import glob

import os

import os





from csst.msc.data import CsstMscImgData

from csst.msc.calib_pos import CsstProcMscPositionCalibration

from csst.msc.calib_pos import CsstProcMscPositionCalibration

from csst.msc.data import CsstMscImgData

from csst.msc.inst_corr import CsstMscInstrumentProc

from csst.msc.inst_corr import CsstMscInstrumentProc





HOSTNAME = os.uname()[1]

HOSTNAME = os.uname()[1]
@@ -29,6 +29,9 @@ elif HOSTNAME == "Dandelion": 
    # gaia catalog directory (for position calibration)

    # gaia catalog directory (for position calibration)

    DIR_GAIA_CATALOG = "/home/csstpipeline/L1Pipeline/msc/gaia_dr3/"

    DIR_GAIA_CATALOG = "/home/csstpipeline/L1Pipeline/msc/gaia_dr3/"





    # only 18 cores available in cloud machine from PMO

    NJOBS = 18



else:

else:

    raise ValueError("Invalid HOSTNAME {}!".format(HOSTNAME))

    raise ValueError("Invalid HOSTNAME {}!".format(HOSTNAME))
@@ -81,7 +84,6 @@ for i_ccd in CCD_ID_LIST: 
    flg_list.append(flg)

    flg_list.append(flg)

    fn_list.append(fp_img)

    fn_list.append(fp_img)







# Step 2. Calibrate Position

# Step 2. Calibrate Position

pcProc = CsstProcMscPositionCalibration()

pcProc = CsstProcMscPositionCalibration()

pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2.0)

pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2.0)
@@ -102,10 +104,18 @@ pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2. 
#     pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2.0)

#     pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2.0)

pcProc.cleanup(img_list, DIR_WORK)

pcProc.cleanup(img_list, DIR_WORK)







# Step 3. Calibrate Flux

# Step 3. Calibrate Flux

from csst.msc.calib_flux import CsstProcFluxCalibration

from csst.msc.calib_flux import CsstProcFluxCalibration

fcProc = CsstProcFluxCalibration()

fcProc = CsstProcFluxCalibration()

# fcProc.prepare()

# fcProc.prepare()

fcProc.run(fn_list,img_list, wht_list, flg_list,wcsdir=DIR_WORK,L1dir=DIR_WORK,workdir=DIR_WORK,refdir=DIR_TEST,addhead=True,morehead=False,plot=False,nodel=False,update=False,upcat=True)

fcProc.run(fn_list, img_list, wht_list, flg_list, wcsdir=DIR_WORK, L1dir=DIR_WORK, workdir=DIR_WORK, refdir=DIR_TEST,

           addhead=True, morehead=False, plot=False, nodel=False, update=False, upcat=True)

fcProc.cleanup(fn_list, DIR_WORK)

fcProc.cleanup(fn_list, DIR_WORK)



# Step 4. Photometry

from csst.msc.phot import CsstMscPhotometryProc

ptProc = CsstMscPhotometryProc()

ptProc.prepare()

ptProc.run(fn_list, out_dir=DIR_WORK, n_jobs=18)

ptProc.cleanup()