"""
Identifier:     KSC-SJ1-MSC-MBI/stats.py
Name:           stats.py
Description:    some stats programs
Author:         Hu Zou (zouhu@nao.cas.cn)
Created:        2022-11-11
Modified-History:
    2022-11-11, created
    2023-12-04, add docstring
    v1.1.2 first version of csst photometry pipeline based on sextractor
    v1.2.0 change weight and flag names due to the CSST naming
           add FLAGS_ISO to remove flagged objects in PSFEx
    v1.2.1 change gain to exposure time
           add options of output images
           change catalog columns to the defined L2 data format
           change config directory
    v1.2.2 add pick_psfstars, select PSF stars
           add photometry types: PSF or MODEL
    v1.2.3 add computing time
    v1.2.4 bug in pick_psfstars with missing max_elp
           bug in psf name if outdir is not provided
           add saturate for phot
    v1.2.5 bug in pick_psfstars, the match_dist should be set
           add PHOT_FLUXFRAC to 0.2,0.5,0.8 to give the radii of containing different fraction of the total flux
    v1.2.6 rename the catalog and clean columns; remove sersic bulge; only use exp disk + dev buldge
           bug in adding multi-PSF fitting, cleaning catalog
    v1.3.0 add ID for single-epoch images defined as obsid+ccdno+objid;
           combine two image header into one primary header in the catalog
           add more annotations
    v1.3.1
           fixing version for the CSST C6
    v1.3.2 coding style fixed
           add return status and file recorder in the do_phot main program (rubish API design)
    v1.3.3 fixed some output defaults and change default bad values
    v1.3.4 add some keywords:VER_, STM_, STA; change output file postfix to upper case; remove PAErr, change ccdno
    v1.3.5 change APCOR? keywords
           change unittest
           change unit test for C6.2
           shrink code
    v1.3.6 CHAGE CODE style
    v1.3.7 CCDCHIP keywords change to CHIPID and value from ccd+number to number
           chang CsstMbiDataManager to CsstMsDataManager
           change all APIs from DM to function
    v1.4.0 new cycle C7:
           get_psf: new limits on FWHM, max_area, PSF size, degree
                    FWHM from [1.5,20] to [1.2,5]
                    max_area from 0 (no limit) to 150
                    psf cutout: from 101 to 31
                    psf size from 101 to 25
                    degree from 3 to 2
                    new strategy for picking psf stars: remove pollution -
                        select good stars - keep max stars if overdensity
    v1.4.1 new API
    v1.4.2 docstring issues
    v1.4.3 add getpsf_flag in API and units in catalog
           add filter in catalog
    v1.4.4 new sextractor background
"""

from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
import numpy as np
import matplotlib.pyplot as plt
import argparse
from astropy import wcs as awcs
from astropy.io import fits
from astropy import table
from astropy.table import Table
from astropy.time import Time
import re
import os
import sys
import time
from scipy.interpolate import UnivariateSpline
from astropy.coordinates import SkyCoord
from shutil import which
import astropy.units as u
from .stats import sigmaclip_limitsig, weighted_mean, closest_match
from .mag_flux_convert import fluxerr2magerr
# from csst_common import CsstResult, CsstStatus
from typing import Optional

prog_dir = os.path.dirname(__file__)
config_path = os.path.join(prog_dir, 'data/')
__version__ = "1.4.3"

# [TODO]
zero_points = {'NUV': 24.8, 'u': 25.22, 'g': 26.23,
               'r': 26.04, 'i': 25.85, 'z': 25.27, 'y': 23.92}


def pick_psfstars(psffile, remove_polution=False, min_nstar=10, max_nstar=1500, class_star=0.7, match_dist=10,
                  min_sn=20, max_elp=0.3, fwhm_range=[1.5, 20]):
    """ pick PSF stars (remove PSF stars with neighbour objects, limit maximum number of stars
    Parameters
    ----------
    psffile str
        catalog used for selecting PSF stars
    remove_polution: bool
        wether to remove objects with pollution from nearby objects
    min_nstar: int
        mininum stars for constructing PSF model
    max_nstar: int
        maximum stars for constructing PSF model
    max_elp: float
        maximum ellpticity
    class_star: float
        class threshhold for PSF stars
    match_dist: float
        matching distance for checking polution in coordinates pixel
    min_sn: float
        minimum S/N for PSF stars

    Returns
    -------
    bool
        whether succeed to pick PSF stars

    """

    # read catalog and keep coordinates
    hdulist = fits.open(psffile, mode='update')
    cat = hdulist[2].data

    if len(cat) < min_nstar:
        print("too few psf stars...")
        hdulist.close()
        return False

    if remove_polution:
        coord = np.array((cat['XWIN_IMAGE'], cat['YWIN_IMAGE'])).transpose()
        coord1 = coord.copy()
        index1, index2 = closest_match(coord, coord1, min_dist=match_dist)
        index1 = np.array(index1)
        mask = np.array([len(ind) == 1 for ind in index2])
        index = index1[mask]
        if len(index) >= min_nstar:
            cat = cat[index]
            print("selecting isolated stars:", len(cat))

    # mask = (cat['flags'] == 0) & (cat['nimaflags_iso'] < 3) & (cat['CLASS_STAR'] > class_star) & (cat['ELLIPTICITY'] < max_elp) & (
    #     cat['SNR_WIN'] > min_sn) & (cat['FWHM_IMAGE'] > fwhm_range[0]) & (cat['FWHM_IMAGE'] < fwhm_range[1])
    # [TODO]
    mask = (cat['flags'] == 0) & (cat['CLASS_STAR'] > class_star) & (cat['ELLIPTICITY'] < max_elp) & (
        cat['SNR_WIN'] > min_sn) & (cat['FWHM_IMAGE'] > fwhm_range[0]) & (cat['FWHM_IMAGE'] < fwhm_range[1])
    cat = cat[mask]

    if len(cat) < min_nstar:
        print("too few psf stars... , got %d" % (len(cat)))
        hdulist.close()
        return False

    if len(cat) > max_nstar:
        print('stars in the catalog, needed:', len(cat), max_nstar)
        if len(cat) > max_nstar:
            indsort = np.argsort(cat['flux_aper'])
            cat = cat[indsort[-max_nstar:]]
    hdulist[2].data = cat
    hdulist.flush()
    hdulist.close()
    print('final picked stars:', len(cat))
    return True
    # if save_starpos:
    # # f=open(fitsname+'-psfxy.txt','w')
    # # for i in range(len(cat)):
    # # 	f.write('%7.2f %7.2f\n' % (cat['XWIN_IMAGE'][i],cat['YWIN_IMAGE'][i]))
    # # f.close()


def get_psf(imgfile, whtfile, flgfile, psffile, psf_size=101, degree=3, variability=0.3, fwhm_range=[2.0, 20.0],
            max_elp=0.3, sampling=0, min_sn=20.0, detect_thresh=5.0, detect_minarea=5, seeing=0.15, pixel_scale=0.075,
            filter_name='gauss_4.0_7x7.conv', phot_aper=10.0, back_size='400,400', nthread=1, **kwd):
    # save_starpos=False
    """ get PSF profile for a specified image
    Parameters
    ----------
    imgfile: str
        input image
    outdir: str
        output directory
    psf_size: int
        size of the PSF image
    degree: int
        order of spatially varied PSF
    variablility: float
        allowed FWHM variability
    fwhm_range: array with two elements
        FWHM range of selected stars
    max_elp: float
        max (A-B)/(A+B) for selected stars
    sampling: float
        sampling step in pixels (0 = Auto)
        min_sn: minimum S/N for souces
    detect_thresh: float
        threshold for detections
    detect_minarea: float
        min pixel area for detections
    seeing: float
        PSF fwhm in arcsec
    pixel_scale: float
        pixel scale in arcsec
    filter_name: str
        convolve kernel file
    phot_aper: float
        aperture used for photometry
    back_size: array
        background grid size
    check_plots: bool
        output the check plots
    nthread: int
        number of threads to get PSF
    **kwd: keywords for PICK_PSFSTARS

    Returns
    -------
    bool, whether succeed to get PSF profile
    """

    # if not system.cmd_exists('sex') or not system.cmd_exists('psfex'):
    if which('sex') is None or which('psfex') is None:
        raise OSError('No sex or psfex SHELL command found! Please install')

    # rootname,_=os.path.splitext(imgfile)
    # indpos=str.rfind(rootname,'_img')
    # flagfile=rootname[:indpos]+'_flg.fits'
    # weightfile=rootname[:indpos]+'_wht.fits'
    # imgfile = dm.l1_detector(detector=detector, post="IMG.fits")
    # whtfile = dm.l1_detector(detector=detector, post="WHT.fits")
    # flgfile = dm.l1_detector(detector=detector, post="FLG.fits")
    # psffile = dm.l1_detector(detector=detector, post="PSF.fits")
    # fitsname = dm.l1_detector(detector=detector, post="")

    head = fits.getheader(imgfile, 0)
    # gain = head['exptime']
    gain = head['EXPTIME']
    saturate = 50000.0 / gain

    # fitsname,_=os.path.splitext(imgfile)
    # fitsname=rootname[:indpos]
    # if outdir is not None:
    # #	 _,filename=os.path.split(rootname[:indpos])
    # #   fitsname=os.path.join(outdir,filename)

    sexfile = os.path.join(config_path, 'csst_psfex.sex')
    covfile = os.path.join(config_path, filter_name)
    nnwfile = os.path.join(config_path, 'default.nnw')
    parfile = os.path.join(config_path, 'csst_psfex.param')
    pexfile = os.path.join(config_path, 'csst.psfex')

    command = 'sex ' + imgfile + ' -c ' + sexfile + ' -CATALOG_NAME ' + psffile + ' -PARAMETERS_NAME ' + parfile \
              + ' -DETECT_MINAREA ' + str(detect_minarea) + ' -DETECT_THRESH ' + str(detect_thresh) + ' -FILTER_NAME ' \
              + covfile + ' -WEIGHT_IMAGE ' + whtfile + ' -FLAG_IMAGE ' + flgfile + ' -PHOT_APERTURES ' \
              + str(phot_aper) + ' -GAIN ' + str(gain) + ' -PIXEL_SCALE ' + str(pixel_scale) + ' -SEEING_FWHM ' \
              + str(seeing) + ' -SATUR_LEVEL ' + str(saturate) + ' -STARNNW_NAME ' + nnwfile + ' -BACK_SIZE ' \
              + back_size + ' -NTHREADS ' + str(nthread)
    print(command)
    os.system(command)

    pickflag = pick_psfstars(psffile, min_sn=min_sn,
                             fwhm_range=fwhm_range, **kwd)
    if pickflag is False:
        if os.path.isfile(psffile):
            os.remove(psffile)
        return False

    check_plots = False
    if not check_plots:
        command = 'psfex ' + psffile + ' -c ' + pexfile + ' -PSF_SIZE ' + str(psf_size) + ',' + str(psf_size) \
                  + ' -SAMPLE_MINSN ' + str(min_sn) + \
                  ' -CHECKPLOT_DEV NULL -CHECKPLOT_TYPE NONE -CHECKIMAGE_TYPE NONE -PSFVAR_DEGREES ' + str(degree) \
                  + ' -SAMPLE_VARIABILITY ' + str(variability) + ' -SAMPLE_FWHMRANGE ' + str(fwhm_range[0]) + ',' \
                  + str(fwhm_range[1]) + ' -PSF_SAMPLING ' + str(sampling) + ' -SAMPLE_MAXELLIP ' + str(max_elp) \
                  + ' -NTHREADS ' + str(nthread)
    # else:
    #    checknames=fitsname+'psffwhm.png,'+fitsname+'psfelp.png'
    #    command='psfex '+psffile+' -c '+pexfile+' -PSF_SIZE '+str(psf_size)+','+str(psf_size)+' -SAMPLE_MINSN '+str(min_sn)+' -PSFVAR_DEGREES '+str(degree)+' -SAMPLE_VARIABILITY '+str(variability)+' -SAMPLE_FWHMRANGE '+str(fwhm_range[0])+','+str(fwhm_range[1])+' -PSF_SAMPLING '+str(sampling) + ' -SAMPLE_MAXELLIP '+str(max_elp)+' -NTHREADS '+str(nthread)+' -CHECKPLOT_TYPE FWHM,ELLIPTICITY -CHECKPLOT_DEV PNG -CHECKPLOT_NAME '+checknames
    print(command)
    os.system(command)
    return True


def photometry(imgfile, whtfile, flgfile, psffile, fluxfile, catfile, skyfile, segfile, detect_thresh=1.0, analysis_thresh=1.0,
               clean='Y', nthread=1, head_zpt=None, aper_cor=True, seeing=0.15,
               pixel_scale=0.075, phot_type='model', sep=2, star_thresh=0.005, plot_flag=False, zp_name='ZP'):
    """
    Perform aperture and model photometry
    Parameters:
    ----------
    imgfile: input image
    outdir: output directory
    detect_thresh: detecting threshold
    analysis_thresh: analysis threshold
    clean: clean the detections
    nthread: number of threads
    head_zpt: flux zeropoint
    aper_cor: wether do aperture correction
    seeing: in arcsec FWHM
    pixel_scale: pixel scale in arcsec
    phot_type: providing the photometry type, if psf, only perform aperture + psf photometry; if model, perform aperture+psf+model photometry
    plot_flag: whether to plot stats
    """

    # fitsname,_=os.path.splitext(imgfile)
    # indpos=str.rfind(fitsname,'_img')
    # fitsname=fitsname[:indpos]
    # if outdir is not None:
    #    if not os.path.exists(outdir):
    #       os.mkdir(outdir)
    #    _,rootname=os.path.split(fitsname)
    #    fitsname=os.path.join(outdir,rootname)

    # imgfile = dm.l1_detector(detector=detector, post="IMG.fits")
    # whtfile = dm.l1_detector(detector=detector, post="WHT.fits")
    # flgfile = dm.l1_detector(detector=detector, post="FLG.fits")
    # psffile = dm.l1_detector(detector=detector, post="PSF.fits")
    # fluxfile = dm.l1_detector(detector=detector, post="FLUX.fits")
    # catfile = dm.l1_detector(detector=detector, post="CAT.fits")
    # fitsname = dm.l1_detector(detector=detector,  post="")

    fitsname, _ = os.path.splitext(imgfile)
    sexfile = os.path.join(config_path, 'csst_phot.sex')
    covfile = os.path.join(config_path, 'default.conv')
    nnwfile = os.path.join(config_path, 'default.nnw')
    if phot_type == "psf":
        parfile = os.path.join(config_path, 'csst_psfphot.params')
    else:
        parfile = os.path.join(config_path, 'csst_modphot.params')
    # psffile = fitsname+'_psf.fits'
    head = fits.getheader(imgfile, 0)
    head1 = fits.getheader(imgfile, 1)
    head.extend(head1)

    if zp_name not in head or head[zp_name] <= 0:
        print("No zeropoint in the header or bad zeropoint")
        # return False
        # [TODO]

    # date=head['date-obs']
    # ccd=head['ccd_no']

    # get flag and weight images
    # flagfile,weightfile=get_flagweght(date,ccd)
    # rootname,_=os.path.splitext(imgfile)
    # indpos=str.rfind(rootname,'_img')
    # flagfile=rootname[:indpos]+'_flg.fits'
    # weightfile=rootname[:indpos]+'_wht.fits'

    # gain = head['exptime']
    gain = head['EXPTIME']
    saturate = 50000.0 / gain

    # types = {'sky': 'BACKGROUND', 'seg': 'SEGMENTATION', 'mod': 'MODELS', 'res': '-MODELS'}
    # catfile = fitsname+'_fluxadu.fits'
    # if checkfiles is None:
    #     checktype = "NONE"
    #     checknames = "check.fits"
    # else:
    #     ntypes = len(checkfiles)
    #     keys = types.keys()
    #     checktype = "'"
    #     checknames = "'"
    #     for i in range(ntypes):
    #         ikey = checkfiles[i]
    #         checktype += types[ikey] + " "
    #         # checknames += fitsname+"_"+ikey+".fits "
    #         checknames += imgfile.replace(".fits", f"_{ikey.upper()}.fits ")
    #     checktype += "'"
    #     checknames += "'"

    # checktype="'BACKGROUND -MODELS MODELS -PSFS PSFS'"
    # checknames='"'+fitsname+'-bkg.fits '+fitsname+'-submodel.fits '+fitsname+'-model.fits '+fitsname+'-subpsf.fits '+fitsname+'-psf.fits '+'"'
    checktype = "'BACKGROUND SEGMENTATION'"
    checknames = "'" + skyfile + " " + segfile + "'"
    command = 'sex ' + imgfile + ' -c ' + sexfile + ' -CATALOG_NAME ' + fluxfile + ' -FILTER_NAME ' + covfile + ' -STARNNW_NAME ' \
              + nnwfile + ' -DETECT_THRESH ' + str(detect_thresh) + ' -ANALYSIS_THRESH ' + str(analysis_thresh) + ' -CLEAN ' \
              + clean + ' -WEIGHT_IMAGE ' + whtfile + ' -FLAG_IMAGE ' + flgfile + ' -PIXEL_SCALE ' + str(pixel_scale) \
              + ' -SEEING_FWHM ' + str(seeing) + ' -GAIN ' + str(gain) + ' -SATUR_LEVEL ' + str(saturate) + ' -PARAMETERS_NAME ' \
              + parfile + ' -CHECKIMAGE_TYPE ' + checktype + ' -CHECKIMAGE_NAME ' + checknames + ' -PSF_NAME ' + psffile + \
              ' -NTHREADS ' + str(nthread)
    print(command)
    os.system(command)
    print("FLUXFILE: ", fluxfile)
    fluxadu = Table.read(fluxfile)
    # calibrate flux and aperture corrections
    fluxcalib = calibrate_fluxadu(
        fluxadu, head, head_zpt=head_zpt, zp_name=zp_name)
    if plot_flag:
        plotname = fitsname
    else:
        plotname = None
    # aperture corrections
    if fluxcalib is not False and aper_cor is True:
        fluxcor = magnitude_correction(fluxcalib, head, elp_lim=0.5, sigma=2.5, plot_name=plotname, magerr_lim=0.1,
                                       sig_limit=0.08, aper_size=[3, 4, 5, 6, 8, 10, 13, 16, 20, 25, 30, 40],
                                       aper_ref=5, class_lim=0.5)
        fluxcor = clean_catalog(fluxcor, head, sep=sep,
                                star_thresh=star_thresh)
        # catfile = fitsname+'_cat.fits'
        fluxcor.write(catfile, format='fits', overwrite=True)
        head['VER_PHOT'] = (__version__, 'version of photometry code')
        nt = Time.now()
        nt.format = 'isot'
        head['SMT_PHOT'] = (nt.value, 'time when photometry ends')
        head['STA_PHOT'] = (0, 'status: 0 good, 1 bad')
        ind_head = head.index('NEXTEND') + 1
        f = fits.open(catfile, mode='update')
        f[0].header.extend(head.cards[ind_head:], bottom=True)
        f.flush(output_verify='fix+warn')
        f.close()
    # if os.path.isfile(catfile):
    #   os.remove(catfile)
    return True


def get_pixscale(head, x, y):
    """ get pixel size at x, y
    Parameters
    ----------
    head: fits.Header
    x: float or array
        image x coordinate
    y: float or array
        image y coordinate

    Returns
    ----------
    pixscales: float, array
        pixel scales in arcsec
    """
    y0 = y - 0.5
    y1 = y + 0.5
    x0 = x - 0.5
    x1 = x + 0.5
    w = awcs.WCS(head)

    ra0, dec0 = w.all_pix2world(x0, y0, 1)
    ra1, dec1 = w.all_pix2world(x0, y1, 1)
    ra2, dec2 = w.all_pix2world(x1, y1, 1)
    ra3, dec3 = w.all_pix2world(x1, y0, 1)

    coord0 = SkyCoord(ra0, dec0, frame='icrs', unit='deg')
    coord1 = SkyCoord(ra1, dec1, frame='icrs', unit='deg')
    coord2 = SkyCoord(ra2, dec2, frame='icrs', unit='deg')
    coord3 = SkyCoord(ra3, dec3, frame='icrs', unit='deg')

    d0 = coord0.separation(coord1).deg
    d1 = coord1.separation(coord2).deg
    d2 = coord2.separation(coord3).deg
    d3 = coord3.separation(coord0).deg
    pixscale = (d0 + d1 + d2 + d3) / 4.0 * 3600.0

    return pixscale


def get_radec_error(head, x, y, xerr, yerr):
    """get ra dec error at x, y assuming the xerr and yerr
    Parameters
    ----------
    head: fits.Header
        fits header
    x: float or array
        image x coordinate
    y: float or array
        image y coordinate
    xerr: float or array
        image x coordinate error
    yerr: float or array
        image coordinate error

    Returns
    ---------
    RAErr: float, array
        RA error
    DECErr: float, array
        DEC error
    """
    y0 = y - yerr
    y1 = y + yerr
    x0 = x - xerr
    x1 = x + xerr
    w = awcs.WCS(head)

    ra0, dec0 = w.all_pix2world(x0, y0, 1)
    ra1, dec1 = w.all_pix2world(x0, y1, 1)
    ra2, dec2 = w.all_pix2world(x1, y1, 1)
    ra3, dec3 = w.all_pix2world(x1, y0, 1)

    allra = np.vstack([ra0, ra1, ra2, ra3])
    alldec = np.vstack([dec0, dec1, dec2, dec3])
    raerr = np.max(allra, axis=0) - np.min(allra, axis=0)
    decerr = np.max(alldec, axis=0) - np.min(alldec, axis=0)
    mask = raerr > 180
    raerr[mask] = -raerr[mask] + 360

    return raerr, decerr


def clean_catalog(cat, head, sep=2.0, star_thresh=0.005):
    """clean catalog, including keeping only one kind of coordinate and remove useless columns, rename the columns to the definition of data products
    Parameters
    ----------
    cat: catalog table
    head: fits.Header
        fits header
    sep: in pixel, the separation between Windowed/Centroid position
    star_thresh: threshold of spread_model to define the stars

    Returns
    ---------
    cat: cleaned catalog
    """
    # bug in sex
    nobj = len(cat)
    cat = table.unique(cat, keys=['X_IMAGE', 'Y_IMAGE'])
    print('duplicate sources:', len(cat) - nobj)
    idx = np.argsort(cat['NUMBER'])
    cat = cat[idx]
    # fix coordinates
    x_cen = np.array(cat['X_IMAGE'])
    y_cen = np.array(cat['Y_IMAGE'])
    xerr_cen = np.sqrt(cat['ERRX2_IMAGE'])
    yerr_cen = np.sqrt(cat['ERRY2_IMAGE'])
    ra_cen = np.array(cat['ALPHA_J2000'])
    dec_cen = np.array(cat['DELTA_J2000'])

    x_win = np.array(cat['XWIN_IMAGE'])
    y_win = np.array(cat['YWIN_IMAGE'])
    xerr_win = np.sqrt(cat['ERRX2WIN_IMAGE'])
    yerr_win = np.sqrt(cat['ERRY2WIN_IMAGE'])
    ra_win = np.array(cat['ALPHAWIN_J2000'])
    dec_win = np.array(cat['DELTAWIN_J2000'])

    x = x_cen.copy()
    xerr = xerr_cen.copy()
    y = y_cen.copy()
    yerr = yerr_cen.copy()
    ra = ra_cen.copy()
    dec = dec_cen.copy()

    mask = (np.abs(x_cen - x_win) < sep) & (np.abs(y_cen - y_win) < sep)
    x[mask] = x_win[mask]
    xerr[mask] = xerr_win[mask]
    y[mask] = y_win[mask]
    yerr[mask] = yerr_win[mask]
    ra[mask] = ra_win[mask]
    dec[mask] = dec_win[mask]
    raerr, decerr = get_radec_error(head, x, y, xerr, yerr)

    xcol = Table.Column(x, name='X', unit='pix')
    xerrcol = Table.Column(xerr, name='XErr', unit='pix')
    ycol = Table.Column(y, name='Y', unit='pix')
    yerrcol = Table.Column(yerr, name='YErr', unit='pix')
    racol = Table.Column(ra, name='RA', unit='deg')
    raerrcol = Table.Column(raerr, name='RAErr', unit='deg')
    deccol = Table.Column(dec, name='DEC', unit='deg')
    decerrcol = Table.Column(decerr, name='DECErr', unit='deg')

    cat.add_columns([xcol, xerrcol, ycol, yerrcol,
                    racol, raerrcol, deccol, decerrcol])
    scales = get_pixscale(head, x, y)

    # calculating Kron radius in arcsec
    k = cat['KRON_RADIUS']
    A = cat['A_IMAGE']
    B = cat['B_IMAGE']
    kronr = k * np.sqrt(A * B) * scales
    cat['KRON_RADIUS'] = kronr
    cat['KRON_RADIUS'].unit = 'arcsec'

    # convert to arcsec
    cat['A_WORLD'] *= 3600
    cat['B_WORLD'] *= 3600
    cat['ERRA_WORLD'] *= 3600
    cat['ERRB_WORLD'] *= 3600
    cat['FWHM_WORLD'] *= 3600
    cat['A_WORLD'].unit = 'arcsec'
    cat['B_WORLD'].unit = 'arcsec'
    cat['ERRA_WORLD'].unit = 'arcsec'
    cat['ERRB_WORLD'].unit = 'arcsec'
    cat['FWHM_WORLD'].unit = 'arcsec'
    for i in range(3):
        cat['FLUX_RADIUS'][:, i] *= scales
    cat['SPHEROID_REFF_WORLD'] *= 3600
    cat['SPHEROID_REFFERR_WORLD'] *= 3600
    cat['SPHEROID_ASPECT_WORLD'] = 1 - cat['SPHEROID_ASPECT_WORLD']
    cat['SPHEROID_REFF_WORLD'].unit = 'arcsec'
    cat['SPHEROID_REFFERR_WORLD'].unit = 'arcsec'
    # effective radius for disk galaxies = 1.67834699 scale length
    cat['DISK_SCALE_WORLD'] *= (3600 * 1.67834699)
    cat['DISK_SCALEERR_WORLD'] *= (3600 * 1.67834699)
    cat['DISK_ASPECT_WORLD'] = 1 - cat['DISK_ASPECT_WORLD']
    cat['DISK_SCALE_WORLD'].unit = 'arcsec'
    cat['DISK_SCALEERR_WORLD'].unit = 'arcsec'

    cols = ['NUMBER', 'X', 'XErr', 'Y', 'YErr', 'RA', 'RAErr', 'DEC', 'DECErr', 'A_WORLD', 'ERRA_WORLD', 'B_WORLD',
            'ERRB_WORLD', 'THETA_WORLD', 'FLAGS', 'IMAFLAGS_ISO', 'NIMAFLAGS_ISO', 'FWHM_WORLD', 'ELONGATION',
            'ELLIPTICITY', 'FLUX_AUTO', 'FLUXERR_AUTO', 'MAG_AUTO', 'MAGERR_AUTO', 'KRON_RADIUS', 'BACKGROUND']
    # re-orgnize aperture photometry and flux radii
    naper = 12
    flux_aper = cat['FLUX_APER']
    fluxerr_aper = cat['FLUXERR_APER']
    mag_aper = cat['MAG_APER']
    magerr_aper = cat['MAGERR_APER']
    for i in range(naper):
        col = Table.Column(cat['FLUX_APER'][:, i],
                           name='Flux_Aper' + str(i + 1), unit='nanomaggy')
        cat.add_column(col)
        col = Table.Column(cat['FLUXERR_APER'][:, i],
                           name='FluxErr_Aper' + str(i + 1), unit='nanomaggy')
        cat.add_column(col)
        col = Table.Column(cat['MAG_APER'][:, i],
                           name='Mag_Aper' + str(i + 1), unit='mag')
        cat.add_column(col)
        col = Table.Column(cat['MAGERR_APER'][:, i],
                           name='MagErr_Aper' + str(i + 1), unit='mag')
        cat.add_column(col)
        cols.extend(['Flux_Aper' + str(i + 1), 'FluxErr_Aper' + str(i + 1), 'Mag_Aper' + str(i + 1),
                     'MagErr_Aper' + str(i + 1)])
    # star-galaxy separation
    gal_type = np.zeros(len(cat), dtype='int32')
    mask = np.abs(cat['SPREAD_MODEL']) > star_thresh
    gal_type[mask] = 1
    col = Table.Column(gal_type, name='Type')
    cat.add_column(col)

    # rename flux radius
    col = Table.Column(cat['FLUX_RADIUS'][:, 0], name='R20', unit='arcsec')
    cat.add_column(col)
    col = Table.Column(cat['FLUX_RADIUS'][:, 1], name='R50', unit='arcsec')
    cat.add_column(col)
    col = Table.Column(cat['FLUX_RADIUS'][:, 2], name='R90', unit='arcsec')
    cat.add_column(col)

    # extract columns to be included in the output catalog
    cols.extend(['Type', 'R20', 'R50', 'R90', 'XPSF_IMAGE', 'YPSF_IMAGE', 'ALPHAPSF_J2000', 'DELTAPSF_J2000',
                 'CHI2_PSF', 'FLUX_PSF', 'FLUXERR_PSF', 'MAG_PSF', 'MAGERR_PSF', 'XMODEL_IMAGE', 'YMODEL_IMAGE',
                 'ALPHAMODEL_J2000', 'DELTAMODEL_J2000', 'CHI2_MODEL', 'FLAGS_MODEL', 'FLUX_MODEL', 'FLUXERR_MODEL',
                 'MAG_MODEL', 'MAGERR_MODEL', 'FLUX_SPHEROID', 'FLUXERR_SPHEROID', 'MAG_SPHEROID', 'MAGERR_SPHEROID',
                 'SPHEROID_REFF_WORLD', 'SPHEROID_REFFERR_WORLD', 'SPHEROID_ASPECT_WORLD', 'SPHEROID_ASPECTERR_WORLD',
                 'SPHEROID_THETA_WORLD', 'SPHEROID_THETAERR_WORLD', 'FLUX_DISK', 'FLUXERR_DISK', 'MAG_DISK',
                 'MAGERR_DISK', 'DISK_SCALE_WORLD', 'DISK_SCALEERR_WORLD', 'DISK_ASPECT_WORLD', 'DISK_ASPECTERR_WORLD',
                 'DISK_THETA_WORLD', 'DISK_THETAERR_WORLD', 'FLUXRATIO_DISK', 'FLUXRATIOERR_DISK', 'SPREAD_MODEL',
                 'SPREADERR_MODEL'])
    cat = cat[cols]

    cols = ['NUMBER', 'A_WORLD', 'ERRA_WORLD', 'B_WORLD', 'ERRB_WORLD', 'THETA_WORLD', 'FLAGS', 'IMAFLAGS_ISO',
            'NIMAFLAGS_ISO', 'FWHM_WORLD', 'ELONGATION', 'ELLIPTICITY', 'FLUX_AUTO', 'FLUXERR_AUTO', 'MAG_AUTO',
            'MAGERR_AUTO', 'KRON_RADIUS', 'BACKGROUND', 'XPSF_IMAGE', 'YPSF_IMAGE', 'ALPHAPSF_J2000', 'DELTAPSF_J2000',
            'CHI2_PSF', 'FLUX_PSF', 'FLUXERR_PSF', 'MAG_PSF', 'MAGERR_PSF', 'XMODEL_IMAGE', 'YMODEL_IMAGE',
            'ALPHAMODEL_J2000', 'DELTAMODEL_J2000', 'CHI2_MODEL', 'FLAGS_MODEL', 'FLUX_MODEL', 'FLUXERR_MODEL',
            'MAG_MODEL', 'MAGERR_MODEL', 'FLUX_SPHEROID', 'FLUXERR_SPHEROID', 'MAG_SPHEROID', 'MAGERR_SPHEROID',
            'SPHEROID_REFF_WORLD', 'SPHEROID_REFFERR_WORLD', 'SPHEROID_ASPECT_WORLD', 'SPHEROID_ASPECTERR_WORLD',
            'SPHEROID_THETA_WORLD', 'SPHEROID_THETAERR_WORLD', 'FLUX_DISK', 'FLUXERR_DISK', 'MAG_DISK', 'MAGERR_DISK',
            'DISK_SCALE_WORLD', 'DISK_SCALEERR_WORLD', 'DISK_ASPECT_WORLD', 'DISK_ASPECTERR_WORLD', 'DISK_THETA_WORLD',
            'DISK_THETAERR_WORLD', 'FLUXRATIO_DISK', 'FLUXRATIOERR_DISK', 'SPREAD_MODEL', 'SPREADERR_MODEL']
    rncols = ['ObjID', 'A', 'AErr', 'B', 'BErr', 'PA', 'Flag', 'Flag_ISO', 'Flag_ISO_Num', 'FWHM', 'AB', 'E',
              'Flux_Kron', 'FluxErr_Kron', 'Mag_Kron', 'MagErr_Kron', 'Radius_Kron', 'Sky', 'X_PSF', 'Y_PSF',
              'RA_PSF', 'DEC_PSF', 'Chi2_PSF', 'Flux_PSF', 'FluxErr_PSF', 'Mag_PSF', 'MagErr_PSF', 'X_Model',
              'Y_Model', 'RA_Model', 'DEC_Model', 'Chi2_Model', 'Flag_Model', 'Flux_Model', 'FluxErr_Model',
              'Mag_Model', 'MagErr_Model', 'Flux_Bulge', 'FluxErr_Bulge', 'Mag_Bulge', 'MagErr_Bulge', 'Re_Bulge',
              'ReErr_Bulge', 'E_Bulge', 'EErr_Bulge', 'PA_Bulge', 'PAErr_Bulge', 'Flux_Disk', 'FluxErr_Disk',
              'Mag_Disk', 'MagErr_Disk', 'Re_Disk', 'ReErr_Disk', 'E_Disk', 'EErr_Disk', 'PA_Disk', 'PAErr_Disk',
              'Ratio_Disk', 'RatioErr_Disk', 'Spread_Model', 'SpreadErr_Model']
    cat.rename_columns(cols, rncols)

    # add IDs
    bits = 6
    objid = np.array(cat['ObjID'])
    if 'CCDCHIP' in head:
        ccdno = int(str.strip(head['CCDCHIP'])[3:])
    else:
        ccdno = int(str.strip(head['CHIPID']))

    obsid = int(head['OBSID'])
    ids = np.array(cat['ObjID'], dtype=int) + ccdno * \
        10 ** bits + obsid * 10 ** (bits + 2)
    col = Table.Column(
        ccdno * np.ones_like(objid).astype('int16'), name='CCDNo')
    cat.add_column(col, index=0)
    col = Table.Column(obsid * np.ones_like(objid).astype('int'), name='ObsID')
    cat.add_column(col, index=0)
    col = Table.Column(ids, name='ID')
    cat.add_column(col, index=0)
    filt = head['FILTER']
    ff = np.zeros(len(cat), dtype='<U3')
    ff[:] = filt
    col = Table.Column(ff, name='Filter')
    cat.add_column(col, index=3)

    # set bad values
    cols = ['FWHM', 'E', 'AB', 'Radius_Kron', 'R20', 'R50', 'R90', 'Re_Bulge', 'ReErr_Bulge', 'E_Bulge', 'EErr_Bulge',
            'Re_Disk', 'ReErr_Disk', 'E_Disk', 'EErr_Disk', 'Ratio_Disk', 'RatioErr_Disk']
    for icol in cols:
        tmpmask = (cat[icol] < 0) | (~np.isfinite(cat[icol]))
        cat[icol][tmpmask] = 0

    keys = ['Kron', 'PSF', 'Model', 'Bulge', 'Disk']
    for ikey in keys:
        mkey = 'Mag_' + ikey
        mekey = 'MagErr_' + ikey
        fkey = 'Flux_' + ikey
        fekey = 'FluxErr_' + ikey
        tmpmask = (cat[mkey] < 0) | (~np.isfinite(cat[mkey])) | (cat[mekey] < 0) | (~np.isfinite(cat[mekey])) | (
            ~np.isfinite(cat[fkey])) | (~np.isfinite(cat[fekey]))
        cat[fkey][tmpmask] = 0
        cat[fekey][tmpmask] = 0
        cat[mkey][tmpmask] = 99.0
        cat[mekey][tmpmask] = 99.0
        cat[fkey].unit = 'nanomaggy'
        cat[fekey].unit = 'nanomaggy'
        cat[mkey].unit = 'mag'
        cat[mekey].unit = 'mag'

    naper = 12
    for i in range(naper):
        ikey = 'Aper' + str(i + 1)
        mkey = 'Mag_' + ikey
        mekey = 'MagErr_' + ikey
        fkey = 'Flux_' + ikey
        fekey = 'FluxErr_' + ikey
        tmpmask = (cat[mkey] < 0) | (~np.isfinite(cat[mkey])) | (cat[mekey] < 0) | (~np.isfinite(cat[mekey])) | (
            ~np.isfinite(cat[fkey])) | (~np.isfinite(cat[fekey]))
        cat[fkey][tmpmask] = 0
        cat[fekey][tmpmask] = 0
        cat[mkey][tmpmask] = 99.0
        cat[mekey][tmpmask] = 99.0

    return cat


def calibrate_fluxadu(fluxadu, head, head_zpt=None, zp_name='ZP'):
    """ Calibrate the original catalog including WCS, flux and others
    head: the image header
    head_zpt: if head_zpt is provided, using this zeropoint instead of that in the header
    """
    # F in nanomaggy = f in adu * 10**((c-22.5)/(-2.5)) where c is zeropoint
    fluxcalib = Table(fluxadu.copy())
    if head_zpt is not None:
        c = head_zpt
    else:
        if zp_name not in head or head[zp_name] <= 0:
            # return False
            # [TODO]
            c = zero_points[head['FILTER']]
        else:
            c = head[zp_name]

    # calibrate flux and errors
    keys = ['APER', 'AUTO', 'HYBRID', 'PSF', 'MODEL', 'SPHEROID', 'DISK', 'MAX_MODEL', 'EFF_MODEL',
            'MEAN_MODEL', 'MAX_SPHEROID', 'EFF_SPHEROID', 'MEAN_SPHEROID', 'MAX_DISK', 'EFF_DISK', 'MEAN_DISK']
    fluxkeys = ['FLUX_' + ikey for ikey in keys]
    fluxerrkeys = ['FLUXERR_' + ikey for ikey in keys]

    nkeys = len(keys)
    colnames = fluxadu.colnames
    for i in range(nkeys):
        if fluxkeys[i] not in colnames:
            continue
        print("calibrate " + fluxkeys[i])
        flux = fluxadu[fluxkeys[i]]
        cflux = np.zeros_like(flux)
        cflux = flux * 10.0 ** ((c - 22.5) / (-2.5))
        fluxcalib[fluxkeys[i]] = cflux
        if i < 7:
            flux_error = fluxadu[fluxerrkeys[i]]
            cflux_error = np.zeros_like(flux)
            cflux_error = flux_error * 10.0 ** ((c - 22.5) / (-2.5))
            fluxcalib[fluxerrkeys[i]] = cflux_error
    return fluxcalib


def magnitude_correction(fluxcalib, head, plot_name=None, magerr_lim=0.05, elp_lim=0.3, sigma=3.0, iters=None,
                         sig_limit=0.1, aper_size=[3, 4, 5, 6, 8, 10, 13, 16, 20, 25, 30, 40], aper_ref=6,
                         class_lim=0.5, min_nstar=5):
    """
    correction for aperture and model magnitudes
    Parameters:
        fluxcalib: the catalog table after calibration
        head: the image header
        plot_name: some plots
        magerr_lim: manigtude error limit to select proper stars
        elp_lim: ellipticity limit
        sigma: sigma for sigma-clipping
        iters: iterations for sigma-clipping
        sig_limit: sigma upper limit
        aper_size: aperture size in radii
        aper_ref: reference aperture for flux calibration
        class_lim: threshhold for class_star
        min_nstar: minimum stars
    Returns
        cat: return the corrected catalog
    """
    low_errlim = 0.005
    fluxcor = fluxcalib.copy()

    elp = fluxcalib['ELLIPTICITY']
    flux = fluxcalib['FLUX_AUTO']
    fluxerr = fluxcalib['FLUXERR_AUTO']
    radius = fluxcalib['FLUX_RADIUS'][:, 1]
    automag, automagerr = fluxerr2magerr(flux, fluxerr)
    tmpmask = (automagerr < low_errlim) & (automagerr > 0)
    automagerr[tmpmask] = low_errlim
    mask_auto = (fluxcalib['FLAGS'] == 0) & (elp < elp_lim) & (automagerr < magerr_lim) & (
        automagerr > 0) & (automag < 90.0) & (radius > 1.0) & (fluxcalib['CLASS_STAR'] > class_lim)

    print('aperture correction for aperture magnitudes ...')
    apersize = np.array(aper_size)
    naper = len(apersize)
    nobj = len(fluxcalib)
    aperstr = ''
    for iaper in apersize:
        aperstr += str(iaper) + ','
    head["apersize"] = (aperstr, 'aperture radii in pixels')
    indref = np.argmin(np.abs(apersize - aper_ref))

    flux = fluxcalib['FLUX_APER']
    fluxerr = fluxcalib['FLUXERR_APER']
    apermag, apermagerr = fluxerr2magerr(flux, fluxerr)
    apmag8 = apermag[:, indref]
    apmag8err = apermagerr[:, indref]
    tmpmask = (apmag8err > 0) & (apmag8err < low_errlim)
    apmag8err[tmpmask] = low_errlim
    mask = mask_auto & (apmag8 > 0.0) & (apmag8 < 90.0) & (
        apmag8err < magerr_lim) & (apmag8err > 0)
    for i in range(naper):
        mask = mask & (apermag[:, i] < 90.0) & (apermag[:, i] > 0.0) & (apermagerr[:, i] < magerr_lim) & (
            apermagerr[:, i] > 0)
    aperflag = True
    apercor = np.zeros(naper)
    apercor_std = np.zeros(naper)
    nstar_aper = 0
    if mask.sum() < min_nstar:
        aperflag = False
        print('not enough stars to do aperture magnitude correction')
    else:
        print('isolated stars: ', mask.sum())
        magdiff = -np.transpose(apermag[mask, :].transpose() - apmag8[mask])
        diff_masked = sigmaclip_limitsig(
            magdiff, sigma=sigma, maxiters=iters, axis=0)
        mask1 = mask
        mask = np.logical_not(np.any(diff_masked.mask, axis=1))
        nstar_aper = mask.sum()
        diff_masked = diff_masked[mask]
        for i in range(naper):
            weighterr = np.sqrt(
                apmag8err[mask1][mask] ** 2 + apermagerr[:, i][mask1][mask] ** 2)
            cor, _, corerr = weighted_mean(
                diff_masked[:, i], weighterr, weight_square=False)
            corerr /= np.sqrt(nstar_aper)
            apercor[i] = cor
            apercor_std[i] = corerr
        # apercor=np.median(diff_masked,axis=0)
        # apercor_std=np.std(diff_masked,axis=0)/np.sqrt(nstar_aper)
        print('aperture correction using stars:', nstar_aper)
        print(np.array([apercor, apercor_std]).transpose())
        if plot_name is not None:
            plt.figure(figsize=(9, 6))
            xsize = apersize.reshape(1, naper).repeat(len(magdiff), axis=0)
            plt.plot(xsize, magdiff, 'k.', markersize=1.0)
            xsize = apersize.reshape(1, naper).repeat(len(diff_masked), axis=0)
            plt.plot(xsize, diff_masked, 'b.', markersize=1.0)
            plt.plot(apersize, apercor, 'r*')
            plt.plot(apersize, apercor, 'r-')
            plt.xlabel('radius (pixels)')
            plt.ylabel('Mag_REF - Mag')
            plt.xlim([0, 43])
            plt.ylim([min(apercor) - 0.2, max(apercor) + 0.2])
            plt.title('aperture correction for aperture magnitudes')
            plt.savefig(plot_name + '-APERcor.png', format='png')
            plt.close()
    head['ns_aper'] = (
        nstar_aper, 'number of stars used in aperture correction')
    for i in np.arange(naper):
        cf = apercor[i]
        ce = apercor_std[i]
        head['apcor' + str(i + 1)] = (cf,
                                      'mag correction for aperture #{}'.format(i))
        head['aperr' + str(i + 1)] = (ce,
                                      'mag correction error for aperture #{}'.format(i))
        ce = 0.0
        ff = fluxcalib['FLUX_APER'][:, i]
        fe = fluxcalib['FLUXERR_APER'][:, i]
        ff1 = ff * 10.0 ** (-cf / 2.5)
        fe1 = np.sqrt(10.0 ** (-2 * cf / 2.5) * fe ** 2 + ff ** 2 * 10.0 ** (-2 * cf / 2.5) * np.log(
            10.0) ** 2 / 2.5 ** 2 * ce ** 2)
        fluxcor['FLUX_APER'][:, i] = ff1
        fluxcor['FLUXERR_APER'][:, i] = fe1
        mag, magerr = fluxerr2magerr(ff1, fe1)
        apermag[:, i] = mag
        apermagerr[:, i] = magerr
    mag_col = Table.Column(apermag, 'MAG_APER')
    magerr_col = Table.Column(apermagerr, 'MAGERR_APER')
    fluxcor.add_column(magerr_col, fluxcor.index_column('FLUXERR_APER') + 1)
    fluxcor.add_column(mag_col, fluxcor.index_column('FLUXERR_APER') + 1)

    # automatic magnitude correction
    # mag correction for kron mag
    print('correct for kron magnitudes ...')
    # mask=mask_auto #automag<90.0
    # if aperflag:
    if True:
        mask = (fluxcor['FLAGS'] == 0)
        # & (elp < elp_lim)
        k = fluxcor['KRON_RADIUS']
        A = fluxcor['A_IMAGE']
        E = fluxcor['ELLIPTICITY']
        B = A * (1.0 - E)
        # B=fluxcor['B_IMAGE']
        kronr = k * np.sqrt(A * B)
        ipt = UnivariateSpline(apersize, apercor, k=3, s=0)
        cf = ipt(kronr)
        ff = fluxcor['FLUX_AUTO']
        fe = fluxcor['FLUXERR_AUTO']
        ff1 = ff * 10.0 ** (-cf / 2.5)
        fluxcor['FLUX_AUTO'] = ff1
        mag, magerr = fluxerr2magerr(ff1, fe)
        mag_col = Table.Column(mag, 'MAG_AUTO')
        magerr_col = Table.Column(magerr, 'MAGERR_AUTO')
        fluxcor.add_column(
            magerr_col, fluxcor.index_column('FLUXERR_AUTO') + 1)
        fluxcor.add_column(mag_col, fluxcor.index_column('FLUXERR_AUTO') + 1)
        print('total kron sources:', mask.sum())
        if plot_name is not None:
            plt.figure(figsize=(9, 6))
            plt.plot(kronr[mask], apmag8[mask] - automag[mask], 'kx')
            plt.plot(apersize, apercor, 'g--')
            rr = np.linspace(0, 43, 100)
            plt.plot(rr, ipt(rr), 'r-')
            plt.xlabel('radius (pixels)')
            plt.ylabel('Mag_REF - Mag')
            plt.xlim([0, 43])
            plt.ylim([min(apercor) - 0.2, max(apercor) + 0.2])
            plt.title('aperture correction for automatic magnitudes')
            plt.savefig(plot_name + '-AUTOcor.png', format='png')
            plt.close()

    # other flux/mag correction
    print('other flux/mag correction: HYBRID PSF MODEL SPHEROID DISK ...')
    keys = ['HYBRID', 'PSF', 'MODEL', 'SPHEROID', 'DISK', 'MAX_MODEL', 'EFF_MODEL', 'MEAN_MODEL', 'MAX_SPHEROID',
            'EFF_SPHEROID', 'MEAN_SPHEROID', 'MAX_DISK', 'EFF_DISK', 'MEAN_DISK']
    fluxkeys = ['FLUX_' + ikey for ikey in keys]
    fluxerrkeys = ['FLUXERR_' + ikey for ikey in keys]
    magkeys = keys.copy()
    for i, ikey in enumerate(keys):
        if i < 5:
            magkeys[i] = 'MAG_' + ikey
        else:
            magkeys[i] = 'MU_' + ikey
    magerrkeys = ['MAGERR_' + ikey for ikey in keys]
    corkeys = ['HYBCOR', 'PSFCOR', 'MODCOR']
    corerrkeys = ['HYBERR', 'PSFERR', 'MODERR']
    nkeys = len(keys)
    colnames = fluxcalib.colnames
    for i in range(nkeys):
        if fluxkeys[i] not in colnames:
            continue
        print('correct ' + fluxkeys[i])
        flux = fluxcalib[fluxkeys[i]]
        if i < 5:
            fluxerr = fluxcalib[fluxerrkeys[i]]
        else:
            fluxerr = np.zeros_like(flux)
        if i < 3:
            kmag, kmagerr = fluxerr2magerr(flux, fluxerr)
            tmpmask = (kmagerr < low_errlim) & (kmagerr > 0)
            kmagerr[tmpmask] = low_errlim
            mask = mask_auto & (kmag < 90.0) & (apmag8 > 0) & (apmag8 < 90.0) & (kmag > 0) & (kmagerr > 0) & (
                kmagerr < magerr_lim) & (apmag8err > 0) & (apmag8err < magerr_lim) & (np.abs(kmag - apmag8) < 0.5)
            corflag = True
            if mask.sum() < min_nstar:
                print('not enough stars to correct ' + magkeys[i])
                cor = 0.0
                corerr = 0.0
                nstar_cor = 0
                corflag = False
            else:
                print('isolated stars for ' + magkeys[i] + ':', mask.sum())
                magdiff = apmag8[mask] - kmag[mask]
                diff_masked = sigmaclip_limitsig(
                    magdiff, sigma=sigma, maxiters=iters, sig_limit=sig_limit)
                mask1 = np.logical_not(diff_masked.mask)
                nstar_cor = mask1.sum()
                diff_masked = magdiff[mask1]
                weighterr = np.sqrt(
                    kmagerr[mask][mask1] ** 2 + apmag8err[mask][mask1] ** 2)
                cor, _, corerr = weighted_mean(
                    diff_masked, weighterr, weight_square=False)
                corerr /= np.sqrt(nstar_cor)
                print('correction using stars:', nstar_cor)
                print([cor, corerr])
            head['ns_' + keys[i]
                 ] = (nstar_cor, 'number of stars used in ' + keys[i] + ' correction')
            head[corkeys[i]] = (cor, 'mag correction for ' + keys[i])
            head[corerrkeys[i]] = (corerr, 'mag correction error')
        cf = cor
        # ce = corerr
        ce = 0.0
        ff = flux
        fe = fluxerr
        ff1 = ff * 10.0 ** (-cf / 2.5)
        fe1 = np.sqrt(10.0 ** (-2 * cf / 2.5) * fe ** 2 + ff ** 2 * 10.0 ** (-2 * cf / 2.5) * np.log(
            10.0) ** 2 / 2.5 ** 2 * ce ** 2)
        fluxcor[fluxkeys[i]] = ff1
        if i < 5:
            fluxcor[fluxerrkeys[i]] = fe1
        mag, magerr = fluxerr2magerr(ff1, fe1)
        mag_col = Table.Column(mag, magkeys[i])
        if i < 5:
            magerr_col = Table.Column(magerr, magerrkeys[i])
            fluxcor.add_column(
                magerr_col, fluxcor.index_column(fluxerrkeys[i]) + 1)
            fluxcor.add_column(
                mag_col, fluxcor.index_column(fluxerrkeys[i]) + 1)
        else:
            fluxcor.add_column(mag_col, fluxcor.index_column(fluxkeys[i]) + 1)
        if corflag and plot_name is not None and i < 3:
            plt.figure(figsize=(9, 6))
            plt.plot(apmag8, apmag8 - kmag, 'kx')
            plt.plot(apmag8[mask], magdiff, 'bx')
            plt.plot(apmag8[mask][mask1], magdiff[mask1], 'r.')
            plt.plot([14, 25], [cor, cor], 'g')
            plt.xlabel('Mag_REF')
            plt.ylabel('Mag_REF-Mag_' + keys[i])
            plt.xlim([14, 25])
            plt.ylim([cor - 0.5, cor + 0.5])
            plt.title('aperture correction for ' + keys[i] + ' magnitudes')
            plt.savefig(plot_name + '-' + keys[i] + 'cor.png', format='png')
            plt.close()
    # add model magnitude and best magnitude
    # print('calculating the model and best magnitudes ...')
    # chi2psf=fluxcalib['CHI2_PSF']
    # chi2mod=fluxcalib['CHI2_MODEL']
    # nobj=len(fluxcor)
    # best_flux=np.zeros(nobj)
    # best_fluxerr=np.zeros(nobj)
    # best_mag=np.zeros(nobj)
    # best_magerr=np.zeros(nobj)
    # best_type=np.repeat(0,nobj)
    # mask=chi2psf < chi2mod
    # best_mag[mask]=fluxcor['MAG_PSF'][mask]
    # best_magerr[mask]=fluxcor['MAGERR_PSF'][mask]
    # best_flux[mask]=fluxcor['FLUX_PSF'][mask]
    # best_fluxerr[mask]=fluxcor['FLUXERR_PSF'][mask]
    # best_mag[~mask]=fluxcor['MAG_MODEL'][~mask]
    # best_magerr[~mask]=fluxcor['MAGERR_MODEL'][~mask]
    # best_flux[~mask]=fluxcor['FLUX_MODEL'][~mask]
    # best_fluxerr[~mask]=fluxcor['FLUXERR_MODEL'][~mask]
    # best_type[~mask]=1
    # bf_col=Table.Column(best_flux,'FLUX_CHI2MIN')
    # bferr_col=Table.Column(best_fluxerr,'FLUXERR_CHI2MIN')
    # bm_col=Table.Column(best_mag,'MAG_CHI2MIN')
    # bmerr_col=Table.Column(best_magerr,'MAGERR_CHI2MIN')
    # btype_col=Table.Column(best_type,'TYPE_CHI2MIN')
    # fluxcor.add_columns([bf_col,bferr_col,bm_col,bmerr_col,btype_col])
    return fluxcor


def rename_columns(cat, keys, rkeys):
    """ rename columns in a table
    Parameters
    ----------
        keys: column names to be renamed
        rkeys: column names named
    Returns
    ----------
        Table, renamed table
    """
    for i in range(len(keys)):
        cat.rename_column(keys[i], rkeys[i])


# new API
def core_msc_l1_mbi_phot(
    image_path: str = "/path/to/image",
    weight_path: str = "/path/to/weight",
    flag_path: str = "/path/to/flag",
    psf_ccds_path: Optional[str] = None,
    psf_local_path: str = "/path/to/psf_local",
    flux_path: str = "/path/to/flux",
    cat_path: str = "/path/to/cat",
    seg_path: str = "/path/to/seg",
    sky_path: str = "/path/to/sky",
    plot_flag: bool = False,
    getpsf_flag: bool = False,
):  # -> CsstResult:
    """
    Generate PSF and do photometry for CSST MBI L1 pipeline.

    Generate PSF using PSFEx and do photometry using SExtractor. If psf_ccds_path is set, use CCDS PSF, else get PSF with PSFEx and then do photometry. If get_psf_flag is set true, force get PSF with PSFEx.

    Parameters
    ----------
    image_path : str
        Path of image, defaulting to "/path/to/image".
    weight_path : str
        Path of weight, defaulting to "/path/to/weight".
    flag_path : str
        Path of flag, defaulting to "/path/to/flag".
    psf_ccds_path : str, optional
        Path of psf_ccds, optional, defaulting to None.
    psf_local_path : str
        Path of psf_local, defaulting to "/path/to/psf_local".
    flux_path : str
        Path of flux adu, defaulting to "/path/to/flux".
    cat_path : str
        Path of catalog, defaulting to "/path/to/cat".
    seg_path : str
        Path of segmentation, defaulting to "/path/to/seg".
    sky_path : str
        Path of sky map, defaulting to "/path/to/sky".
    plot_flag : bool
        Wether output some diagnostic plots, default False.
    getpsf_flag : bool
        Force to get PSF, default False.

    Returns
    -------
    CsstResult
        Result containing `status`, `file_list`, and `output`.
    """

    time0 = time.time()
    # generate psf
    if psf_ccds_path is None or getpsf_flag:
        print("get PSF model for " + image_path)
        pflag = generate_psf(image_path, weight_path,
                             flag_path, psf_local_path)
        time1 = time.time()
        psftime = time1 - time0

    # pick a PSF
    if psf_ccds_path is None:
        # use psf_path
        psf_path = psf_local_path
    else:
        psf_path = psf_ccds_path

    if os.path.isfile(psf_path):
        # photometry
        print('do photometry for ' + image_path)
        do_photometry(
            image_path,
            weight_path,
            flag_path,
            psf_path,
            flux_path,
            cat_path,
            sky_path,
            seg_path,
            plot_flag,
        )

        time2 = time.time()
        if 'time1' in vars().keys():
            phottime = time2 - time1
        else:
            phottime = time2 - time0

        print('Total time for ' + image_path + ': ', time2 - time0)
        if os.path.isfile(cat_path):
            f = fits.open(cat_path, mode='update')
            if 'psftime' in vars().keys():
                f[1].header['PSF_PATH'] = psf_path
                f[1].header['PSFTIME'] = (psftime, 'seconds')
                f[1].header['PHOTTIME'] = (phottime, 'seconds')
                f.flush(output_verify='fix+warn')
                f.close()

    # # construct CsstResult
    # result = CsstResult(
    #     status=CsstStatus(0 if os.path.exists(cat_path) else 2),
    #     files=[
    #         psf_path,
    #         flux_path,
    #         cat_path,
    #         seg_path,
    #         sky_path,
    #     ],
    # )
    # return result


def generate_psf(
    image_path: str = "/path/to/image",
    weight_path: str = "/path/to/weight",
    flag_path: str = "/path/to/flag",
    psf_local_path: str = "/path/to/psf_local",
) -> None:
    """
    Generate psf for csst image.

    Parameters
    ----------
    image_path : str
        Path of image, defaulting to "/path/to/image".
    weight_path : str
        Path of weight, defaulting to "/path/to/weight".
    flag_path : str
        Path of flag, defaulting to "/path/to/flag".
    psf_local_path : str
        Path of psf_local..., defaulting to "/path/to/psf_local".

    Returns
    -------
    None
    """
    get_psf(image_path, weight_path, flag_path, psf_local_path, psf_size=25, degree=2, variability=0.3,
            fwhm_range=[1.1, 4.0], max_elp=0.3, sampling=0, min_sn=5.0, detect_thresh=5.0, detect_minarea=3,
            back_size='500,500', nthread=0, remove_polution=True, min_nstar=10, max_nstar=1500, class_star=0.7,
            match_dist=10.0)


def do_photometry(
    image_path: str = "/path/to/image",
    weight_path: str = "/path/to/weight",
    flag_path: str = "/path/to/flag",
    psf_path: str = "/path/to/psf",
    flux_path: str = "/path/to/flux",
    cat_path: str = "/path/to/cat",
    sky_path: str = "/path/to/sky",
    seg_path: str = "/path/to/seg",
    plot_flag: bool = False,
) -> None:
    """
    Do photometry for csst image.

    Parameters
    ----------
    image_path : str
        Path of image, defaulting to "/path/to/image".
    weight_path : str
        Path of weight, defaulting to "/path/to/weight".
    flag_path : str
        Path of flag, defaulting to "/path/to/flag".
    psf_path : str
        Path of psf, defaulting to "/path/to/psf".
    flux_path : str
        Path of flux, defaulting to "/path/to/flux".
    cat_path : str
        Path of cat, defaulting to "/path/to/cat".
    sky_path : str
        Path of sky, defaulting to "/path/to/sky".
    seg_path : str
        Path of seg, defaulting to "/path/to/seg".
    plot_flag : bool
        Wether output some diagnostic plots, default Fasle.

    Returns
    -------
    None
    """
    photometry(image_path, weight_path, flag_path, psf_path, flux_path, cat_path, sky_path, seg_path,
               detect_thresh=1.0, analysis_thresh=1.0, clean='Y',
               head_zpt=None, nthread=0, phot_type='model',
               sep=2, star_thresh=0.005, zp_name='ZP', plot_flag=plot_flag)
