init

from concurrent import futures

from glob import glob

from astropy.io import fits

import os

import numpy as np

from time import time

from concurrent.futures import ThreadPoolExecutor

A, B = 2, 2

def split(data: np.ndarray, i: int) -> np.ndarray:

    h, w = data.shape

    x, y = h//A, w//B

    n, m = i//B, i % B

    data = data[n*x:(n+1)*x,

                m*y:(m+1)*y].copy()

    return data

def concatenate(data_list):

    data_u = np.concatenate(data_list[:B], axis=1)

    data_d = np.concatenate(data_list[B:], axis=1)

    data = np.concatenate([data_u, data_d], axis=0)

    return data

def array_combine(ndarray, mode="mean") -> np.ndarray:

    """ Function to combine 3-D data array

    Parameters

    ----------

    ndarray: array, input data cube (3D)

    model: mean, median, sum, mean_clip, median_clip, default is mean

    """

    if mode == "median":

        array = np.median(ndarray, axis=0)

    elif mode == "median_clip":

        ndarray = np.sort(ndarray, axis=0)[1:-1]

        array = np.median(ndarray, axis=0)

    elif mode == "sum":

        array = np.sum(ndarray, axis=0)

    elif mode == "mean":

        array = np.mean(ndarray, axis=0)

    elif mode == "mean_clip":

        ndarray = np.sort(ndarray, axis=0)[1:-1]

        array = np.mean(ndarray, axis=0)

    return array

def load_bias(path: str, i: int) -> np.ndarray:

    with fits.open(path) as hdul:

        du = hdul[1].data

    du = du.astype(int)

    du = split(du, i)

    return du

def load_dark(path: str, i: int, bias: np.ndarray) -> np.ndarray:

    with fits.open(path) as hdul:

        du = hdul[1].data

        hu = hdul[0].header

    du = du.astype(int)

    du = du - bias

    du = du / hu["EXPTIME"]

    du = split(du, i)

    return du

def load_flat(path: str, i: int, bias: np.ndarray, dark: np.ndarray) -> np.ndarray:

    with fits.open(path) as hdul:

        du = hdul[1].data

        hu = hdul[0].header

    du = du.astype(int)

    du = du - bias - dark * hu["EXPTIME"]

    du = du / hu["EXPTIME"]

    du = du / np.median(du)

    du = split(du, i)

    return du

def save_fits(func, mode: str, path_list, save_path, *args) -> np.ndarray:

    ch_list = []

    with ThreadPoolExecutor() as tpe:

        for i in range(A*B):

            futures = [tpe.submit(func, path, i, *args) for path in path_list]

            du_list = [future.result() for future in futures]

            du = array_combine(du_list, mode)

            ch_list.append(du)

    du = concatenate(ch_list)

    du_output = os.path.basename(path_list[0])

    du_output = du_output.replace("raw", "combine")

    du_output = os.path.join(save_path, du_output)

    du = du.astype(np.float32)

    du_fits = fits.HDUList([fits.PrimaryHDU(data=du)])

    du_fits.writeto(du_output, overwrite=True)

    return du

def main(input_path : str, save_path : str, number_list : list):

    for number in number_list:

        print(number, end=' ')

        bias_path = glob(os.path.join(input_path, "MSC*/*CLB*_" + number + '_*'))

        bias = save_fits(load_bias, "median", bias_path, save_path)

        print('bias finish', end=' ')

        dark_path = glob(os.path.join(input_path, "MSC*/*CLD*_" + number + '_*'))

        dark = save_fits(load_dark, "median", dark_path, save_path, bias)

        print('dark finish', end=' ')

        flat_path = glob(os.path.join(input_path, "MSC*/*CLF*_" + number + '_*'))

        flat = save_fits(load_flat, "median", flat_path, save_path, bias, dark)

        print('flat finish')

if __name__ == "__main__":

    input_path = "/data/test20211012/ref/"

    save_path = "/data/test20211012/ref/"

    number_list = ['06', '07', '08', '09', '11', '12', '13', '14',

                   '15', '16', '17', '18', '19', '20', '22', '23', '24', '25']

    main(input_path, save_path, number_list)

FROM hub.cstcloud.cn/csst/python:3.8

RUN pip install --no-deps ccdproc && pip install deepCR

RUN apt-get update && apt-get install -y vim

RUN mkdir -p /home/csstpipeline/code/csst-msc-instrument-effect/csst_msc_instrument_effect

# COPY run.sh /app/bin/

COPY instrument_effect_wrapper.py setup.py requirements.txt \ 

    /home/csstpipeline/code/csst-msc-instrument-effect/

COPY csst_msc_instrument_effect/* \ 

    /home/csstpipeline/code/csst-msc-instrument-effect/csst_msc_instrument_effect/

COPY MSC_crmask.ini /home/csstpipeline/csst-msc-instrument-effect-master/

RUN cd /home/csstpipeline/code/csst-msc-instrument-effect/ \

    && python setup.py install

COPY cleanup.sh /app/bin/

COPY csst_msc_iec.sh /app/bin/run.sh

[global]

;which method used for cosmic ray detection

;model=deepCR

update_flag=True

;whether save CR cleaned image

save_flag=True

;flag image suffix

flag_suffix=flg

;CR masked image suffix

data_suffix=crclean

;whether append CR masked image to the output FITS file

append_flag=False

;mask image suffix

mask_suffix=crmask

;whether use GPU

gpu_flag=False

;whether fill appropriate value for CR contaminated pixels

fill_flag=True

;filling method

fill_method=meanmask

;torch thread

torch_thread=1

[deepCR]

threshold=0.5

inpaint=True

binary=True

patch=256

segment=True

parallel=True

;Cosmic ray clean training model

clean_model=/home/csstpipeline/csst-msc-instrument-effect-main/CSST_2021-12-30_CCD23_epoch20.pth

;Inpaint model

inpaint_model=ACS-WFC-F606W-2-32

n_jobs=4

;train parameters

;ignore pixel, e.g., bad pixel, saturation, npy or fits

;ignore=ignore.fits

;sky=sky.fits

;aug_sky_min=0

;aug_sky_max=0

;name=xxxx

hidden=50

gpu=False

epoch=50

batch=16

lr=0.005

auto_lr_decay=True

lr_decay_patience=4

lr_decay_factor=0.1

save_after=1

plot_every=10

use_tqdm=False

use_tqdm_notebook=False

plot_roc=False

directory=./data/

[lacosmic]

sigclip=4.5

sigfrac=0.3

objlim=5.0

gain=1.0

readnoise=6.5

satlevel=65535.0

pssl=0.0

niter=4

sepmed=True

cleantype=meanmask

fsmode=median

psfmodel=gauss

psffwhm=2.5

psfsize=7

psfbeta=4.765

gain_apply=True

IMAGE=hub.cstcloud.cn/csst/detector-effect-correction

build:

	docker build --network=host -t $(IMAGE) .

push:

	docker push $(IMAGE)

clean:

	docker rmi $(IMAGE)

rmi:

	for H in h0 h1 h2 ; do echo $$H ; ssh $$H docker rmi $(IMAGE) ; done