reformated flux calibration module

#!/usr/bin/env python3.8

#!/usr/bin/python3.8

#Written by ZZM

#Edited on Jun. 17, 2016

#add color term

__author__ = 'ZZM'

import sys

import os

import time

from subprocess import Popen

import shutil

import numpy as np

from astropy import table

from astropy import units as u

from astropy.coordinates import SkyCoord

from astropy.io import fits

from astropy.stats import sigma_clip

from astropy.stats import sigma_clipped_stats

from astropy.io import fits

from astropy.wcs import WCS

from astropy import table

from multiprocessing import Pool

from subprocess import Popen, PIPE

from astropy.wcs.utils import proj_plane_pixel_scales

import time

from .. import PACKAGE_PATH

from ..core.processor import CsstProcessor

#---------------------------------------------------------------------------

# Edited on Jun. 17, 2016

# add color term

__author__ = 'ZZM'

class CsstProcFluxCalibration(CsstProcessor):

    def __init__(self):

        pass

    def __init__(self, **kwargs):

        super().__init__(**kwargs)

    def ps1_mags(self, cat, band, obs_region):

        # get ps1 color-corrected magnitude for specific filter

        return mags, magerr, ra, dec, brmedian

    #----------------------------------------------------------------------

    def read_ps1cat(self, ra, dec, outcat, path='/line12/Pan-STARRS/chunks-qz-star-v2/', silent=True):

        # Read a piece of the PS1 calibration catalog containing ra,dec

        from healpy import ang2pix

                ps = table.vstack(ps)

        if outcat: ps.write(outcat, format='fits', overwrite=True)

        return ps

    #-----------------------------------------------------------------------

    def read_gaiacat(self, ra, dec, outcat, path='./', silent=True):

        # function: Read a piece of the GAIA calibration catalog containing ra,dec

        from healpy import ang2pix

        if npix > 8:

            print('too many healpix files:', npix, ' check image wcs!')

            return

        dt=np.dtype([('source_id', '>i8'), ('ra', '>f8'), ('dec', '>f8'), ('ra_error', '>f4'), ('dec_error', '>f4'), ('phot_g_mean_mag', '>f4'), ('phot_g_mean_flux_over_error', '>f4'), ('phot_g_n_obs', '>i2'), ('phot_bp_mean_mag', '>f4'), ('phot_bp_mean_flux_over_error', '>f4'), ('phot_bp_n_obs', '>i2'), ('phot_rp_mean_mag', '>f4'), ('phot_rp_mean_flux_over_error', '>f4'), ('phot_rp_n_obs', '>i2'), ('astrometric_weight_al', '>f4'), ('astrometric_n_obs_al', '>i2'), ('astrometric_n_good_obs_al', '>i2'), ('astrometric_excess_noise', '>f4'), ('astrometric_excess_noise_sig', '>f4'), ('duplicated_source', '?'), ('ref_epoch', '>f4'), ('parallax', '>f4'), ('parallax_error', '>f4'), ('pmra', '>f4'), ('pmra_error', '>f4'), ('pmdec', '>f4'), ('pmdec_error', '>f4'), ('phot_variable_flag', '?')])

        dt = np.dtype([('source_id', '>i8'), ('ra', '>f8'), ('dec', '>f8'), ('ra_error', '>f4'), ('dec_error', '>f4'),

                       ('phot_g_mean_mag', '>f4'), ('phot_g_mean_flux_over_error', '>f4'), ('phot_g_n_obs', '>i2'),

                       ('phot_bp_mean_mag', '>f4'), ('phot_bp_mean_flux_over_error', '>f4'), ('phot_bp_n_obs', '>i2'),

                       ('phot_rp_mean_mag', '>f4'), ('phot_rp_mean_flux_over_error', '>f4'), ('phot_rp_n_obs', '>i2'),

                       ('astrometric_weight_al', '>f4'), ('astrometric_n_obs_al', '>i2'),

                       ('astrometric_n_good_obs_al', '>i2'), ('astrometric_excess_noise', '>f4'),

                       ('astrometric_excess_noise_sig', '>f4'), ('duplicated_source', '?'), ('ref_epoch', '>f4'),

                       ('parallax', '>f4'), ('parallax_error', '>f4'), ('pmra', '>f4'), ('pmra_error', '>f4'),

                       ('pmdec', '>f4'), ('pmdec_error', '>f4'), ('phot_variable_flag', '?')])

        ps = table.Table(dtype=dt)

        for i in pix:

            fname = path + 'chunk-' + '%5.5d' % i + '.fits'

                ps = table.vstack(ps)

        if outcat: ps.write(outcat, format='fits', overwrite=True)

        return ps

    ##########################

    def gaia_mags(self, cat, band, obs_region):

        # get Gaia color-corrected magnitude for specific filter

        refc = SkyCoord(ra, dec, unit=(u.deg, u.deg), frame='fk5')

        return mags, magerr, gimedian, refc

    #########################################

    def read_inputcat(self, image, outcat='refcat.fits', refdir='', silent=True):

        # imname=os.path.split(image)[-1]

        data.write(outcat, format='fits', overwrite=True)

        return data

    ###########################

    def input_mags(self, cat, usepixcood=False):

        goodid = (cat['mag'] < 20) & (cat['mag'] > 16)

        brmedian = 0.0

        return mags, magerr, brmedian, refc

    #########################################

    def rewrite_sex_cat(self, cat, workdir=''):

        data = fits.open(cat)

                    hdu.writeto(subfile, overwrite=True)

                index.append(filename)

        return index

    ##########################

    def split_wcs_head(self, wcshead, im_index=[], workdir=''):

        if len(im_index) == 0:

            wheader = head[i].header

            prihdu = fits.PrimaryHDU(header=wheader)

            prihdu.writeto(headname, overwrite=True)

    ##########################

    def run_sextractor(self, image, cat):

        image0 = image

        p.wait()

        return cat

    ##########################

    def prepare(self, image, wcsdir, workdir, usewcsresult=False, newcat=False):

        # make calibration files: Obs. cat (SExtractor), PSF model (PSFex)

        header = self.combine_head(image, wcshead1, wcshead2, prime=False)

        return wcscat1, cat, ref, header

    #-----------------------------------------------------------------------

    # def getebv(image):

    #    ebv=0.0

    #    if tid.sum()==1: ebv=tiles['EBV'][tid].data[0]

    #    #header.set('ebv',round(ebv,4),'E(B-V) from SFD1998')

    #    return ebv

    #########################

    def getmlim(self, fwhm=0.15, avsky=7.0, rdnoise=5.0, zpt=25.8, ebv=0.0, filter='g'):

        # E(B-V): 3.995, 3.214, 2.165, 1.592, 1.211, 1.064 for ugrizY decals

        flim = noise * snr

        mlim = -2.5 * np.log10(flim) + zpt - k * ebv

        return mlim

    #########################

    def getfwhm(self, fwhmsex, ellip, obsme, obsflags):

        nanid = np.isnan(fwhmsex)

        # print id,fwhmid.data[~fwhmid.mask][id]

        # print fwhmid, fwhmsex

        return fwhm

    #########################

    def makedatedir(self, path):

        index = path.rfind('/')

        f1.close()

        f.close()

        return wcshead

    ##########################

    def combine_head(self, image, wcshead1='', wcshead2='', prime=False):

        # combine image head and wcs head keywords

            h0['CTYPE2'] = 'DEC--TPV'

        return h0

    ##########################

    def match_calib(self, obsc, refc, obsm, refm, obsme, refme, obsflags, fwhmsex=np.array([])):

        # matching obs and ref catalogs

                obs_uidlim = obs_uid[uidlim]

                ref_uidlim = ref_uid[uidlim]

                ccdraoff=np.median((obsc[obs_uidlim].ra- refc[ref_uidlim].ra).arcsec*np.cos(obsc[obs_uidlim].dec.deg*np.pi/180))

                ccdraoff = np.median(

                    (obsc[obs_uidlim].ra - refc[ref_uidlim].ra).arcsec * np.cos(obsc[obs_uidlim].dec.deg * np.pi / 180))

                ccddecoff = np.median((obsc[obs_uidlim].dec - refc[ref_uidlim].dec).arcsec)

                # if fwhmsex.size >1:

                #    fwhm=np.median(fwhmsex[obs_uidlim])

        # print(coeff0,cstd,csize,cobsm,crefm,ccdraoff,ccddecoff)

        # print('crefm.min,max=',refm.min(),refm.max())

        return coeff0, cstd, csize, cobsm, crefm, ccdraoff, ccddecoff, fwhm

    ########################

    def calib(self,image,imgdata,whtdata,flgdata,wcsdir='./',L1dir='./',workdir='./',refdir='',addhead=False,morehead=True,plot=False,nodel=True,update=False,upcat=True):

    def calib(self, image, imgdata, whtdata, flgdata, wcsdir='./', L1dir='./', workdir='./', refdir='', addhead=False,

              morehead=True, plot=False, nodel=True, update=False, upcat=True):

        print('calibration for:', image)

        if not os.path.exists(L1dir):

        vernum = 'FluxCalib_v1.0'

        vernum_com = 'version of calibration code'

        #keys=['cali_ref','ccdzp','ccdzpa','ccdzpb','ccdzpc','ccdzpd','ccdphoff','ccdphrms', 'phrmsA','phrmsB','phrmsC','phrmsD','aper_r','fwhm','seeing','raoff','decoff','trans','ccdnstar','nmatch','nmatcha','nmatchb','nmatchc','nmatchd', 'mdncol','colt_par','ebv','EXTNAME','cali_v']

        # keys=['cali_ref','ccdzp','ccdzpa','ccdzpb','ccdzpc','ccdzpd','ccdphoff','ccdphrms', 'phrmsA','phrmsB',

        # 'phrmsC','phrmsD','aper_r','fwhm','seeing','raoff','decoff','trans','ccdnstar','nmatch','nmatcha',

        # 'nmatchb','nmatchc','nmatchd', 'mdncol','colt_par','ebv','EXTNAME','cali_v']

        # tmpa=round(coeff0[0],4)

        #########################

        ##set header keywords: flux calibration information##

        # set header keywords: flux calibration information##

        header.set('cali_ref', cali_ref, cali_ref_com)

        header.set('COMMENT', '=' * 66, before='cali_ref')

        header.set('COMMENT', 'Flux calibration information', before='cali_ref')

        # header.set('colt_par',colortpar,colortpar_com)

        # header.set('ebv',round(ebv,4),'E(B-V) from SFD1998')

        ###Calculate and set SKY & magnitude limiting#####################

        # Calculate and set SKY & magnitude limiting

        if not ('SKYRMS' in list(header.keys())):

            # imdata=fits.getdata(image, 0)

            imdata = imgdata[1].data

        mlim = self.getmlim(fwhm=fwhm, avsky=avsky, rdnoise=5.0, zpt=ccdzp, ebv=ebv, filter=band)

        mlim_com = 'magnitude limiting of 5-sigma galaxy detection'

        header.set('mlim', round(mlim, 2), mlim_com)

        #######set signals of calibration progress################

        # set signals of calibration progress

        opetime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

        header.set('FLUX_S', 0, 'flux calibration status')

        header.set('FLUX_V', '1.3', vernum_com)

        # close the QC1 list file

        qc1.close()

        #################plot######################

        # plot

        if plot:

            print('plot calibration chart ...')

            import matplotlib

            plt.savefig(psname)

        return coeff, std, match

#---------------------------------------------------------------------------

    def run(self,fn_list,img_list=[], wht_list=[], flg_list=[],wcsdir='./',L1dir='./',workdir='./',refdir='',addhead=True,morehead=False,plot=False,nodel=True,update=False,upcat=True):

    def run(self, fn_list, img_list=[], wht_list=[], flg_list=[], wcsdir='./', L1dir='./', workdir='./', refdir='',

            addhead=True, morehead=False, plot=False, nodel=True, update=False, upcat=True):

        if len(fn_list) == 0:

            print('Flux calibration: No input images in img_list!')

            if not os.path.isfile(image):

                print(('cannot find the file:' + image))

            else:

                self.calib(image,imgdata,whtdata,flgdata,wcsdir=wcsdir,L1dir=L1dir,workdir=workdir,refdir=refdir,addhead=addhead,morehead=morehead,plot=plot,nodel=nodel,update=update)

                self.calib(image, imgdata, whtdata, flgdata, wcsdir=wcsdir, L1dir=L1dir, workdir=workdir, refdir=refdir,

                           addhead=addhead, morehead=morehead, plot=plot, nodel=nodel, update=update)

        # time2=time.time()

        print('\n############### flux calibration done #############\n')

######################################

    def cleanup(self, fn_list, workdir, nodel=False):

        # clean up environment

        for image in fn_list:

            if not nodel:

                try:

                    os.remove(cat)

                except(FileNotFoundError): 

                except FileNotFoundError:

                    print()

                try:

                    os.remove(ref)

                except(FileNotFoundError): 

                except FileNotFoundError:

                    print()

                try:

                    os.remove(whead)

                except(FileNotFoundError): 

                except FileNotFoundError:

                    print()