Commit ba060b49 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

updated pipeline

parent 732d8334

csst/msc/phot.py

+6 −5
Original line number Diff line number Diff line 
from pathlib import Path



import joblib

import numpy as np

from ccdproc import cosmicray_lacosmic

from deepCR import deepCR
@@ -13,19 +13,20 @@ from .. import PACKAGE_PATH 
# DEEPCR_MODEL_PATH = PACKAGE_PATH + "/msc/deepcr_model/CSST_2021-12-30_CCD23_epoch20.pth"





class CsstMscInPhotometryProc(CsstProcessor):

class CsstMscPhotometryProc(CsstProcessor):



    def __init__(self):

        super(CsstProcessor, self).__init__()



    def prepare(self, **kwargs):

        # check whether SExtractor and PSFEx exists

        pass



    def cleanup(self):

        pass



    def run(self, fl_in, out_dir):

    def run(self, fl_in, out_dir, n_jobs=1, verbose=5):

        # assert len(fl_in) == len(fl_out)

        for i in range(len(fl_in)):

            do_phot(fl_in[i], outdir=out_dir)

        joblib.Parallel(n_jobs=n_jobs, verbose=verbose)(

            joblib.delayed(do_phot)(fl_in[i], outdir=out_dir) for i in range(len(fl_in)))

        return

csst/msc/pipeline.py

+16 −6
Original line number Diff line number Diff line
@@ -3,8 +3,8 @@ 
import glob

import os



from csst.msc.data import CsstMscImgData

from csst.msc.calib_pos import CsstProcMscPositionCalibration

from csst.msc.data import CsstMscImgData

from csst.msc.inst_corr import CsstMscInstrumentProc



HOSTNAME = os.uname()[1]
@@ -29,6 +29,9 @@ elif HOSTNAME == "Dandelion": 
    # gaia catalog directory (for position calibration)

    DIR_GAIA_CATALOG = "/home/csstpipeline/L1Pipeline/msc/gaia_dr3/"



    # only 18 cores available in cloud machine from PMO

    NJOBS = 18



else:

    raise ValueError("Invalid HOSTNAME {}!".format(HOSTNAME))
@@ -81,7 +84,6 @@ for i_ccd in CCD_ID_LIST: 
    flg_list.append(flg)

    fn_list.append(fp_img)





# Step 2. Calibrate Position

pcProc = CsstProcMscPositionCalibration()

pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2.0)
@@ -102,10 +104,18 @@ pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2. 
#     pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2.0)

pcProc.cleanup(img_list, DIR_WORK)





# Step 3. Calibrate Flux

from csst.msc.calib_flux import CsstProcFluxCalibration

fcProc = CsstProcFluxCalibration()

# fcProc.prepare()

fcProc.run(fn_list,img_list, wht_list, flg_list,wcsdir=DIR_WORK,L1dir=DIR_WORK,workdir=DIR_WORK,refdir=DIR_TEST,addhead=True,morehead=False,plot=False,nodel=False,update=False,upcat=True)

fcProc.run(fn_list, img_list, wht_list, flg_list, wcsdir=DIR_WORK, L1dir=DIR_WORK, workdir=DIR_WORK, refdir=DIR_TEST,

           addhead=True, morehead=False, plot=False, nodel=False, update=False, upcat=True)

fcProc.cleanup(fn_list, DIR_WORK)



# Step 4. Photometry

from csst.msc.phot import CsstMscPhotometryProc

ptProc = CsstMscPhotometryProc()

ptProc.prepare()

ptProc.run(fn_list, out_dir=DIR_WORK, n_jobs=18)

ptProc.cleanup()