diff --git a/csst/msc/phot.py b/csst/msc/phot.py index 6289302a8d023cb1a5c66821bc16604f68d82a94..eac545788767ff84cba18954cc423e9d4d5e57fa 100644 --- a/csst/msc/phot.py +++ b/csst/msc/phot.py @@ -1,5 +1,5 @@ from pathlib import Path - +import joblib import numpy as np from ccdproc import cosmicray_lacosmic from deepCR import deepCR @@ -13,19 +13,20 @@ from .. import PACKAGE_PATH # DEEPCR_MODEL_PATH = PACKAGE_PATH + "/msc/deepcr_model/CSST_2021-12-30_CCD23_epoch20.pth" -class CsstMscInPhotometryProc(CsstProcessor): +class CsstMscPhotometryProc(CsstProcessor): def __init__(self): super(CsstProcessor, self).__init__() def prepare(self, **kwargs): + # check whether SExtractor and PSFEx exists pass def cleanup(self): pass - def run(self, fl_in, out_dir): + def run(self, fl_in, out_dir, n_jobs=1, verbose=5): # assert len(fl_in) == len(fl_out) - for i in range(len(fl_in)): - do_phot(fl_in[i], outdir=out_dir) + joblib.Parallel(n_jobs=n_jobs, verbose=verbose)( + joblib.delayed(do_phot)(fl_in[i], outdir=out_dir) for i in range(len(fl_in))) return diff --git a/csst/msc/pipeline.py b/csst/msc/pipeline.py index 4de9779edcf4af861290744b3dbd8e4972a158b4..174204ce71da147b9da6063c9aa887f82eff102d 100644 --- a/csst/msc/pipeline.py +++ b/csst/msc/pipeline.py @@ -3,8 +3,8 @@ import glob import os -from csst.msc.data import CsstMscImgData from csst.msc.calib_pos import CsstProcMscPositionCalibration +from csst.msc.data import CsstMscImgData from csst.msc.inst_corr import CsstMscInstrumentProc HOSTNAME = os.uname()[1] @@ -29,6 +29,9 @@ elif HOSTNAME == "Dandelion": # gaia catalog directory (for position calibration) DIR_GAIA_CATALOG = "/home/csstpipeline/L1Pipeline/msc/gaia_dr3/" + # only 18 cores available in cloud machine from PMO + NJOBS = 18 + else: raise ValueError("Invalid HOSTNAME {}!".format(HOSTNAME)) @@ -81,7 +84,6 @@ for i_ccd in CCD_ID_LIST: flg_list.append(flg) fn_list.append(fp_img) - # Step 2. Calibrate Position pcProc = CsstProcMscPositionCalibration() pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2.0) @@ -102,10 +104,18 @@ pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2. # pcProc.run(img_list, wht_list, flg_list, fn_list, DIR_GAIA_CATALOG, DIR_WORK, 2.0) pcProc.cleanup(img_list, DIR_WORK) - # Step 3. Calibrate Flux from csst.msc.calib_flux import CsstProcFluxCalibration fcProc = CsstProcFluxCalibration() -#fcProc.prepare() -fcProc.run(fn_list,img_list, wht_list, flg_list,wcsdir=DIR_WORK,L1dir=DIR_WORK,workdir=DIR_WORK,refdir=DIR_TEST,addhead=True,morehead=False,plot=False,nodel=False,update=False,upcat=True) -fcProc.cleanup(fn_list,DIR_WORK) +# fcProc.prepare() +fcProc.run(fn_list, img_list, wht_list, flg_list, wcsdir=DIR_WORK, L1dir=DIR_WORK, workdir=DIR_WORK, refdir=DIR_TEST, + addhead=True, morehead=False, plot=False, nodel=False, update=False, upcat=True) +fcProc.cleanup(fn_list, DIR_WORK) + +# Step 4. Photometry +from csst.msc.phot import CsstMscPhotometryProc +ptProc = CsstMscPhotometryProc() +ptProc.prepare() +ptProc.run(fn_list, out_dir=DIR_WORK, n_jobs=18) +ptProc.cleanup() +