Chip.py 52.1 KB
Newer Older
Fang Yuedong's avatar
Fang Yuedong committed
1
2
3
import galsim
import os
import numpy as np
Fang Yuedong's avatar
Fang Yuedong committed
4
5
import pickle
import json
6
import ObservationSim.Instrument._util as _util
Fang Yuedong's avatar
Fang Yuedong committed
7
8
from astropy.table import Table
from numpy.random import Generator, PCG64
Fang Yuedong's avatar
Fang Yuedong committed
9
10
from astropy.io import fits
from datetime import datetime
Fang Yuedong's avatar
Fang Yuedong committed
11

12
13
14
from ObservationSim.Instrument.Chip import Effects as effects
from ObservationSim.Instrument.FocalPlane import FocalPlane
from ObservationSim.Config.Header import generatePrimaryHeader, generateExtensionHeader
Fang Yuedong's avatar
Fang Yuedong committed
15
from ObservationSim.Instrument._util import rotate_conterclockwise
16
from ObservationSim.Instrument.Chip import ChipUtils as chip_utils
17

Wei Chengliang's avatar
Wei Chengliang committed
18
19
from ObservationSim.Instrument.Chip.libCTI.CTI_modeling import CTI_sim

Fang Yuedong's avatar
Fang Yuedong committed
20
21
22
23
24
25
try:
    import importlib.resources as pkg_resources
except ImportError:
    # Try backported to PY<37 'importlib_resources'
    import importlib_resources as pkg_resources

Fang Yuedong's avatar
Fang Yuedong committed
26

Fang Yuedong's avatar
Fang Yuedong committed
27
class Chip(FocalPlane):
28
    def __init__(self, chipID, ccdEffCurve_dir=None, CRdata_dir=None, sls_dir=None, config=None, treering_func=None, logger=None):
Fang Yuedong's avatar
Fang Yuedong committed
29
30
        # Get focal plane (instance of paraent class) info
        super().__init__()
31
32
        self.nsecy = 2
        self.nsecx = 8
Fang Yuedong's avatar
Fang Yuedong committed
33
        self.gain_channel = np.ones(self.nsecy * self.nsecx)
Fang Yuedong's avatar
Fang Yuedong committed
34
        self._set_attributes_from_config(config)
Fang Yuedong's avatar
Fang Yuedong committed
35

36
37
        self.logger = logger

Fang Yuedong's avatar
Fang Yuedong committed
38
39
        # A chip ID must be assigned
        self.chipID = int(chipID)
Fang Yuedong's avatar
Fang Yuedong committed
40
        self.chip_name = str(chipID).rjust(2, '0')
Fang Yuedong's avatar
Fang Yuedong committed
41
42
43
44
45

        # Get corresponding filter info
        self.filter_id, self.filter_type = self.getChipFilter()
        self.survey_type = self._getSurveyType()

Fang Yuedong's avatar
Fang Yuedong committed
46
47
48
        if self.filter_type != "FGS":
            self._getChipRowCol()

49
        # Set the relavent specs for detectors
Fang Yuedong's avatar
Fang Yuedong committed
50
51
52
53
54
55
56
57
58
59
        try:
            with pkg_resources.files('ObservationSim.Instrument.data.ccd').joinpath("chip_definition.json") as chip_definition:
                with open(chip_definition, "r") as f:
                    chip_dict = json.load(f)[str(self.chipID)]
        except AttributeError:
            with pkg_resources.path('ObservationSim.Instrument.data.ccd', "chip_definition.json") as chip_definition:
                with open(chip_definition, "r") as f:
                    chip_dict = json.load(f)[str(self.chipID)]
        for key in chip_dict:
            setattr(self, key, chip_dict[key])
Fang Yuedong's avatar
Fang Yuedong committed
60

61
        self.fdModel = None
Fang Yuedong's avatar
Fang Yuedong committed
62
        if self.filter_type == "FGS":
63
64
            fgs_name = self.chip_name[0:4]
            try:
Fang Yuedong's avatar
Fang Yuedong committed
65
                with pkg_resources.files('ObservationSim.Instrument.data.field_distortion').joinpath("FieldDistModelGlobal_pr4_%s.pickle" % (fgs_name.lower())) as field_distortion:
66
67
68
                    with open(field_distortion, "rb") as f:
                        self.fdModel = pickle.load(f)
            except AttributeError:
Fang Yuedong's avatar
Fang Yuedong committed
69
                with pkg_resources.path('ObservationSim.Instrument.data.field_distortion', "FieldDistModelGlobal_pr4_%s.pickle" % (fgs_name.lower())) as field_distortion:
70
71
                    with open(field_distortion, "rb") as f:
                        self.fdModel = pickle.load(f)
Fang Yuedong's avatar
Fang Yuedong committed
72
73
        else:
            # Get the corresponding field distortion model
74
75
76
77
78
79
80
81
            try:
                with pkg_resources.files('ObservationSim.Instrument.data.field_distortion').joinpath("FieldDistModel_v2.0.pickle") as field_distortion:
                    with open(field_distortion, "rb") as f:
                        self.fdModel = pickle.load(f)
            except AttributeError:
                with pkg_resources.path('ObservationSim.Instrument.data.field_distortion', "FieldDistModelGlobal_mainFP_v1.0.pickle") as field_distortion:
                    with open(field_distortion, "rb") as f:
                        self.fdModel = pickle.load(f)
Fang Yuedong's avatar
Fang Yuedong committed
82

Fang Yuedong's avatar
Fang Yuedong committed
83
84
        # Get boundary (in pix)
        self.bound = self.getChipLim()
Fang Yuedong's avatar
Fang Yuedong committed
85

Fang Yuedong's avatar
Fang Yuedong committed
86
87
        self.ccdEffCurve_dir = ccdEffCurve_dir
        self.CRdata_dir = CRdata_dir
Fang Yuedong's avatar
Fang Yuedong committed
88

89
        slsconfs = chip_utils.getChipSLSConf(chipID=self.chipID)
Fang Yuedong's avatar
Fang Yuedong committed
90
        if np.size(slsconfs) == 1:
91
92
93
94
95
96
            try:
                with pkg_resources.files('ObservationSim.Instrument.data.sls_conf').joinpath(slsconfs) as conf_path:
                    self.sls_conf = str(conf_path)
            except AttributeError:
                with pkg_resources.path('ObservationSim.Instrument.data.sls_conf', slsconfs) as conf_path:
                    self.sls_conf = str(conf_path)
Fang Yuedong's avatar
Fang Yuedong committed
97
        else:
Fang Yuedong's avatar
Fang Yuedong committed
98
99
            # self.sls_conf = [os.path.join(self.sls_dir, slsconfs[0]), os.path.join(self.sls_dir, slsconfs[1])]
            self.sls_conf = []
100
101
102
103
104
105
106
107
108
109
110
111
            try:
                with pkg_resources.files('ObservationSim.Instrument.data.sls_conf').joinpath(slsconfs[0]) as conf_path:
                    self.sls_conf.append(str(conf_path))
            except AttributeError:
                with pkg_resources.path('ObservationSim.Instrument.data.sls_conf', slsconfs[0]) as conf_path:
                    self.sls_conf.append(str(conf_path))
            try:
                with pkg_resources.files('ObservationSim.Instrument.data.sls_conf').joinpath(slsconfs[1]) as conf_path:
                    self.sls_conf.append(str(conf_path))
            except AttributeError:
                with pkg_resources.path('ObservationSim.Instrument.data.sls_conf', slsconfs[1]) as conf_path:
                    self.sls_conf.append(str(conf_path))
Fang Yuedong's avatar
Fang Yuedong committed
112

Fang Yuedong's avatar
Fang Yuedong committed
113
114
115
        self.effCurve = self._getChipEffCurve(self.filter_type)
        self._getCRdata()

116
117
        # # Define the sensor model
        self.sensor = galsim.Sensor()
Fang Yuedong's avatar
Fang Yuedong committed
118

Fang Yuedong's avatar
Fang Yuedong committed
119
        self.flat_cube = None  # for spectroscopic flat field cube simulation
120

Fang Yuedong's avatar
Fang Yuedong committed
121
122
123
124
125
    def _set_attributes_from_config(self, config):
        # Default setting
        self.read_noise = 5.0   # e/pix
        self.dark_noise = 0.02  # e/pix/s
        self.rotate_angle = 0.
Fang Yuedong's avatar
Fang Yuedong committed
126
        self.overscan = 1000
Fang Yuedong's avatar
Fang Yuedong committed
127
        # Override default values
128
129
130
        # for key in ["gain", "bias_level, dark_exptime", "flat_exptime", "readout_time", "full_well", "read_noise", "dark_noise", "overscan"]:
        #     if key in config["ins_effects"]:
        #         setattr(self, key, config["ins_effects"][key])
Fang Yuedong's avatar
Fang Yuedong committed
131

Fang Yuedong's avatar
Fang Yuedong committed
132
133
134
135
136
137
138
139
140
    def _getChipRowCol(self):
        self.rowID, self.colID = self.getChipRowCol(self.chipID)

    def getChipRowCol(self, chipID):
        rowID = ((chipID - 1) % 5) + 1
        colID = 6 - ((chipID - 1) // 5)
        return rowID, colID

    def _getSurveyType(self):
141
        if self.filter_type in _util.SPEC_FILTERS:
Fang Yuedong's avatar
Fang Yuedong committed
142
            return "spectroscopic"
143
        elif self.filter_type in _util.PHOT_FILTERS:
Fang Yuedong's avatar
Fang Yuedong committed
144
            return "photometric"
Fang Yuedong's avatar
Fang Yuedong committed
145
146
        # elif self.filter_type in ["FGS"]:
        #     return "FGS"
Fang Yuedong's avatar
Fang Yuedong committed
147
148
149

    def _getChipEffCurve(self, filter_type):
        # CCD efficiency curves
Fang Yuedong's avatar
Fang Yuedong committed
150
151
152
153
154
155
156
        if filter_type in ['NUV', 'u', 'GU']:
            filename = 'UV0.txt'
        if filter_type in ['g', 'r', 'GV', 'FGS']:
            # TODO, need to switch to the right efficiency curvey for FGS CMOS
            filename = 'Astro_MB.txt'
        if filter_type in ['i', 'z', 'y', 'GI']:
            filename = 'Basic_NIR.txt'
157
158
159
160
161
162
        try:
            with pkg_resources.files('ObservationSim.Instrument.data.ccd').joinpath(filename) as ccd_path:
                table = Table.read(ccd_path, format='ascii')
        except AttributeError:
            with pkg_resources.path('ObservationSim.Instrument.data.ccd', filename) as ccd_path:
                table = Table.read(ccd_path, format='ascii')
Fang Yuedong's avatar
Fang Yuedong committed
163
164
        throughput = galsim.LookupTable(
            x=table['col1'], f=table['col2'], interpolant='linear')
Fang Yuedong's avatar
Fang Yuedong committed
165
166
167
168
        bandpass = galsim.Bandpass(throughput, wave_type='nm')
        return bandpass

    def _getCRdata(self):
169
170
171
172
173
174
        try:
            with pkg_resources.files('ObservationSim.Instrument.data').joinpath("wfc-cr-attachpixel.dat") as cr_path:
                self.attachedSizes = np.loadtxt(cr_path)
        except AttributeError:
            with pkg_resources.path('ObservationSim.Instrument.data', "wfc-cr-attachpixel.dat") as cr_path:
                self.attachedSizes = np.loadtxt(cr_path)
175

Fang Yuedong's avatar
Fang Yuedong committed
176
177
178
179
180
181
182
183
184
185
186
187
188
189
    # def loadSLSFLATCUBE(self, flat_fn='flat_cube.fits'):
    #     try:
    #         with pkg_resources.files('ObservationSim.Instrument.data').joinpath(flat_fn) as data_path:
    #             flat_fits = fits.open(data_path, ignore_missing_simple=True)
    #     except AttributeError:
    #         with pkg_resources.path('ObservationSim.Instrument.data', flat_fn) as data_path:
    #             flat_fits = fits.open(data_path, ignore_missing_simple=True)

    #     fl = len(flat_fits)
    #     fl_sh = flat_fits[0].data.shape
    #     assert fl == 4, 'FLAT Field Cube is Not 4 layess!!!!!!!'
    #     self.flat_cube = np.zeros([fl, fl_sh[0], fl_sh[1]])
    #     for i in np.arange(0, fl, 1):
    #         self.flat_cube[i] = flat_fits[i].data
Fang Yuedong's avatar
Fang Yuedong committed
190

191
    def getChipFilter(self, chipID=None):
Fang Yuedong's avatar
Fang Yuedong committed
192
193
        """Return the filter index and type for a given chip #(chipID)
        """
194
        filter_type_list = _util.ALL_FILTERS
Fang Yuedong's avatar
Fang Yuedong committed
195
196
197
198
        if chipID == None:
            chipID = self.chipID

        # updated configurations
Fang Yuedong's avatar
Fang Yuedong committed
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
        if chipID > 42 or chipID < 1:
            raise ValueError("!!! Chip ID: [1,42]")
        if chipID in [6, 15, 16, 25]:
            filter_type = "y"
        if chipID in [11, 20]:
            filter_type = "z"
        if chipID in [7, 24]:
            filter_type = "i"
        if chipID in [14, 17]:
            filter_type = "u"
        if chipID in [9, 22]:
            filter_type = "r"
        if chipID in [12, 13, 18, 19]:
            filter_type = "NUV"
        if chipID in [8, 23]:
            filter_type = "g"
        if chipID in [1, 10, 21, 30]:
            filter_type = "GI"
        if chipID in [2, 5, 26, 29]:
            filter_type = "GV"
        if chipID in [3, 4, 27, 28]:
            filter_type = "GU"
        if chipID in range(31, 43):
            filter_type = 'FGS'
Fang Yuedong's avatar
Fang Yuedong committed
223
224
225
226
227
228
        filter_id = filter_type_list.index(filter_type)

        return filter_id, filter_type

    def getChipLim(self, chipID=None):
        """Calculate the edges in pixel for a given CCD chip on the focal plane
Fang Yuedong's avatar
Fang Yuedong committed
229
        NOTE: There are 5*4 CCD chips in the focus plane for photometric / spectroscopic observation.
Fang Yuedong's avatar
Fang Yuedong committed
230
231
232
233
234
235
        Parameters:
            chipID:         int
                            the index of the chip
        Returns:
            A galsim BoundsD object
        """
236
237
238
239
240
241
242
243
        xmin, xmax, ymin, ymax = 1e10, -1e10, 1e10, -1e10
        xcen = self.x_cen / self.pix_size
        ycen = self.y_cen / self.pix_size
        sign_x = [-1., 1., -1., 1.]
        sign_y = [-1., -1., 1., 1.]
        for i in range(4):
            x = xcen + sign_x[i] * self.npix_x / 2.
            y = ycen + sign_y[i] * self.npix_y / 2.
Fang Yuedong's avatar
Fang Yuedong committed
244
245
            x, y = _util.rotate_conterclockwise(
                x0=xcen, y0=ycen, x=x, y=y, angle=self.rotate_angle)
246
247
248
            xmin, xmax = min(xmin, x), max(xmax, x)
            ymin, ymax = min(ymin, y), max(ymax, y)
        return galsim.BoundsD(xmin, xmax, ymin, ymax)
Fang Yuedong's avatar
Fang Yuedong committed
249
250

    def getSkyCoverage(self, wcs):
Fang Yuedong's avatar
Fang Yuedong committed
251
        # print("In getSkyCoverage: xmin = %.3f, xmax = %.3f, ymin = %.3f, ymax = %.3f"%(self.bound.xmin, self.bound.xmax, self.bound.ymin, self.bound.ymax))
Fang Yuedong's avatar
Fang Yuedong committed
252
253
254
255
256
257
258
259
260
        return super().getSkyCoverage(wcs, self.bound.xmin, self.bound.xmax, self.bound.ymin, self.bound.ymax)

    def getSkyCoverageEnlarged(self, wcs, margin=0.5):
        """The enlarged sky coverage of the chip
        """
        margin /= 60.0
        bound = self.getSkyCoverage(wcs)
        return galsim.BoundsD(bound.xmin - margin, bound.xmax + margin, bound.ymin - margin, bound.ymax + margin)

Fang Yuedong's avatar
Fang Yuedong committed
261
    def isContainObj(self, ra_obj=None, dec_obj=None, x_image=None, y_image=None, wcs=None, margin=1):
Fang Yuedong's avatar
Fang Yuedong committed
262
        # magin in number of pix
Fang Yuedong's avatar
Fang Yuedong committed
263
264
265
        if (ra_obj is not None) and (dec_obj is not None):
            if wcs is None:
                wcs = self.img.wcs
Fang Yuedong's avatar
Fang Yuedong committed
266
267
            pos_obj = wcs.toImage(galsim.CelestialCoord(
                ra=ra_obj*galsim.degrees, dec=dec_obj*galsim.degrees))
Fang Yuedong's avatar
Fang Yuedong committed
268
269
            x_image, y_image = pos_obj.x, pos_obj.y
        elif (x_image is None) or (y_image is None):
Fang Yuedong's avatar
Fang Yuedong committed
270
271
            raise ValueError(
                "Either (ra_obj, dec_obj) or (x_image, y_image) should be given")
Fang Yuedong's avatar
Fang Yuedong committed
272

Fang Yuedong's avatar
Fang Yuedong committed
273
274
        xmin, xmax = self.bound.xmin - margin, self.bound.xmax + margin
        ymin, ymax = self.bound.ymin - margin, self.bound.ymax + margin
Fang Yuedong's avatar
Fang Yuedong committed
275
        if (x_image - xmin) * (x_image - xmax) > 0.0 or (y_image - ymin) * (y_image - ymax) > 0.0:
Fang Yuedong's avatar
Fang Yuedong committed
276
277
278
279
280
281
282
            return False
        return True

    def getChipNoise(self, exptime=150.0):
        noise = self.dark_noise * exptime + self.read_noise**2
        return noise

Zhang Xin's avatar
Zhang Xin committed
283
    def addEffects(self, config, img, chip_output, filt, ra_cen, dec_cen, img_rot, exptime=150., pointing_ID=0, timestamp_obs=1621915200, pointing_type='SCI', sky_map=None, post_flash_map=None, tel=None, logger=None):
284
        # Set random seeds
Fang Yuedong's avatar
Fang Yuedong committed
285
286
        SeedGainNonuni = int(config["random_seeds"]["seed_gainNonUniform"])
        SeedBiasNonuni = int(config["random_seeds"]["seed_biasNonUniform"])
Fang Yuedong's avatar
Fang Yuedong committed
287
288
289
290
        SeedRnNonuni = int(config["random_seeds"]["seed_rnNonUniform"])
        SeedBadColumns = int(config["random_seeds"]["seed_badcolumns"])
        SeedDefective = int(config["random_seeds"]["seed_defective"])
        SeedCosmicRay = int(config["random_seeds"]["seed_CR"])
Fang Yuedong's avatar
Fang Yuedong committed
291
        fullwell = int(self.full_well)
Fang Yuedong's avatar
Fang Yuedong committed
292
        if config["ins_effects"]["add_hotpixels"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
293
294
295
            BoolHotPix = True
        else:
            BoolHotPix = False
Fang Yuedong's avatar
Fang Yuedong committed
296
        if config["ins_effects"]["add_deadpixels"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
297
298
299
            BoolDeadPix = True
        else:
            BoolDeadPix = False
300
        self.logger = logger
Fang Yuedong's avatar
Fang Yuedong committed
301

302
        # Get Poisson noise generator
303
304
        rng_poisson, poisson_noise = chip_utils.get_poisson(
            seed=int(config["random_seeds"]["seed_poisson"]) + pointing_ID*30 + self.chipID, sky_level=0.)
305

306
        # Add sky background
Fang Yuedong's avatar
Fang Yuedong committed
307
        if config["ins_effects"]["add_back"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
308
309
            img, sky_map = chip_utils.add_sky_background(
                img=img, filt=filt, exptime=exptime, sky_map=sky_map, tel=tel)
310
            del sky_map
311

Fang Yuedong's avatar
Fang Yuedong committed
312
        # Apply flat-field large scale structure for one chip
Fang Yuedong's avatar
Fang Yuedong committed
313
        if config["ins_effects"]["flat_fielding"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
314
315
            chip_utils.log_info(
                msg="  Creating and applying Flat-Fielding", logger=self.logger)
316
            chip_utils.log_info(msg=str(img.bounds), logger=self.logger)
Fang Yuedong's avatar
Fang Yuedong committed
317
318
            flat_img, flat_normal = chip_utils.get_flat(
                img=img, seed=int(config["random_seeds"]["seed_flat"]))
319
320
            if self.survey_type == "photometric":
                img *= flat_normal
Fang Yuedong's avatar
Fang Yuedong committed
321
            del flat_normal
Fang Yuedong's avatar
Fang Yuedong committed
322
            if config["output_setting"]["flat_output"] == False:
Fang Yuedong's avatar
Fang Yuedong committed
323
324
                del flat_img

Zhang Xin's avatar
Zhang Xin committed
325
326
327
        if post_flash_map is not None:
            img = img + post_flash_map

Fang Yuedong's avatar
Fang Yuedong committed
328
        # Apply Shutter-effect for one chip
Fang Yuedong's avatar
Fang Yuedong committed
329
        if config["ins_effects"]["shutter_effect"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
330
331
332
333
334
            chip_utils.log_info(
                msg="  Apply shutter effect", logger=self.logger)
            # shutter effect normalized image for this chip
            shuttimg = effects.ShutterEffectArr(
                img, t_shutter=1.3, dist_bearing=735, dt=1E-3)
335
336
            if self.survey_type == "photometric":
                img *= shuttimg
Fang Yuedong's avatar
Fang Yuedong committed
337
338
            # output 16-bit shutter effect image with pixel value <=65535
            if config["output_setting"]["shutter_output"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
339
                shutt_gsimg = galsim.ImageUS(shuttimg*6E4)
Fang Yuedong's avatar
Fang Yuedong committed
340
341
                shutt_gsimg.write("%s/ShutterEffect_%s_1.fits" %
                                  (chip_output.subdir, self.chipID))
Fang Yuedong's avatar
Fang Yuedong committed
342
343
                del shutt_gsimg
            del shuttimg
344
345
346
347
348
        # # Add Poisson noise to the resulting images
        # # (NOTE): this can only applied to the slitless image
        # # since it dose not use photon shooting to draw stamps
        # if self.survey_type == "spectroscopic":
        #     img.addNoise(poisson_noise)
Fang Yuedong's avatar
Fang Yuedong committed
349
350

        # Add cosmic-rays
Fang Yuedong's avatar
Fang Yuedong committed
351
        if config["ins_effects"]["cosmic_ray"] == True and pointing_type == 'SCI':
352
            chip_utils.log_info(msg="  Adding Cosmic-Ray", logger=self.logger)
Fang Yuedong's avatar
Fang Yuedong committed
353
354
            img, crmap_gsimg, cr_event_num = chip_utils.add_cosmic_rays(img=img, chip=self, exptime=exptime,
                                                                        seed=SeedCosmicRay+pointing_ID*30+self.chipID)
355
356
            chip_utils.outputCal(
                chip=self,
Fang Yuedong's avatar
Fang Yuedong committed
357
358
359
360
361
362
363
                img=crmap_gsimg,
                ra_cen=ra_cen,
                dec_cen=dec_cen,
                img_rot=img_rot,
                im_type='CRS',
                pointing_ID=pointing_ID,
                output_dir=chip_output.subdir,
364
                exptime=exptime,
365
366
                project_cycle=config["project_cycle"],
                run_counter=config["run_counter"],
367
                timestamp=timestamp_obs)
Fang Yuedong's avatar
Fang Yuedong committed
368
369
            del crmap_gsimg

370
        # Apply PRNU effect and output PRNU flat file:
Fang Yuedong's avatar
Fang Yuedong committed
371
        if config["ins_effects"]["prnu_effect"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
372
373
374
375
            chip_utils.log_info(
                msg="  Applying PRNU effect", logger=self.logger)
            img, prnu_img = chip_utils.add_PRNU(img=img, chip=self,
                                                seed=int(config["random_seeds"]["seed_prnu"]+self.chipID))
Fang Yuedong's avatar
Fang Yuedong committed
376
            if config["output_setting"]["prnu_output"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
377
378
                prnu_img.write("%s/FlatImg_PRNU_%s.fits" %
                               (chip_output.subdir, self.chipID))
Fang Yuedong's avatar
Fang Yuedong committed
379
            if config["output_setting"]["flat_output"] == False:
380
381
                del prnu_img

382
383
384
385
386
387
        # # Add dark current
        # if config["ins_effects"]["add_dark"] == True:
        #     dark_noise = galsim.DeviateNoise(galsim.PoissonDeviate(rng_poisson, self.dark_noise*(exptime+0.5*self.readout_time)))
        #     img.addNoise(dark_noise)

        # Add dark current & Poisson noise
388
        InputDark = False
Fang Yuedong's avatar
Fang Yuedong committed
389
        if config["ins_effects"]["add_dark"] == True:
390
            if InputDark:
Fang Yuedong's avatar
Fang Yuedong committed
391
392
                img = chip_utils.add_inputdark(
                    img=img, chip=self, exptime=exptime)
393
            else:
Fang Yuedong's avatar
Fang Yuedong committed
394
395
                img, _ = chip_utils.add_poisson(
                    img=img, chip=self, exptime=exptime, poisson_noise=poisson_noise)
396
        else:
Fang Yuedong's avatar
Fang Yuedong committed
397
398
            img, _ = chip_utils.add_poisson(
                img=img, chip=self, exptime=exptime, poisson_noise=poisson_noise, dark_noise=0.)
399
400
401
402

        # Add diffusion & brighter-fatter effects
        if config["ins_effects"]["bright_fatter"] == True:
            img = chip_utils.add_brighter_fatter(img=img)
403
404
405
406

        # Add Hot Pixels or/and Dead Pixels
        rgbadpix = Generator(PCG64(int(SeedDefective+self.chipID)))
        badfraction = 5E-5*(rgbadpix.random()*0.5+0.7)
Fang Yuedong's avatar
Fang Yuedong committed
407
408
        img = effects.DefectivePixels(img, IfHotPix=BoolHotPix, IfDeadPix=BoolDeadPix,
                                      fraction=badfraction, seed=SeedDefective+self.chipID, biaslevel=0)
409

Fang Yuedong's avatar
Fang Yuedong committed
410
        # Apply Bad lines
Fang Yuedong's avatar
Fang Yuedong committed
411
        if config["ins_effects"]["add_badcolumns"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
412
413
            img = effects.BadColumns(
                img, seed=SeedBadColumns, chipid=self.chipID, logger=self.logger)
414
415

        # Apply Nonlinearity on the chip image
Fang Yuedong's avatar
Fang Yuedong committed
416
        if config["ins_effects"]["non_linear"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
417
418
            chip_utils.log_info(
                msg="  Applying Non-Linearity on the chip image", logger=self.logger)
419
420
421
            img = effects.NonLinearity(GSImage=img, beta1=5.e-7, beta2=0)

        # Apply CCD Saturation & Blooming
Fang Yuedong's avatar
Fang Yuedong committed
422
        if config["ins_effects"]["saturbloom"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
423
424
425
426
            chip_utils.log_info(
                msg="  Applying CCD Saturation & Blooming", logger=self.logger)
            img = effects.SaturBloom(
                GSImage=img, nsect_x=1, nsect_y=1, fullwell=fullwell)
427
428

        # Apply CTE Effect
Fang Yuedong's avatar
Fang Yuedong committed
429
430
431
        # if config["ins_effects"]["cte_trail"] == True:
        # chip_utils.log_info(msg="  Apply CTE Effect", logger=self.logger)
        # img = effects.CTE_Effect(GSImage=img, threshold=27)
Wei Chengliang's avatar
Wei Chengliang committed
432

Fang Yuedong's avatar
Fang Yuedong committed
433
434
435
436
        pre1 = self.prescan_x  # 27
        over1 = self.overscan_x  # 71
        pre2 = self.prescan_y  # 0 #4
        over2 = self.overscan_y  # 84 #80
Wei Chengliang's avatar
Wei Chengliang committed
437

Fang Yuedong's avatar
Fang Yuedong committed
438
        if config["ins_effects"]["cte_trail"] == True:
439
            chip_utils.log_info(msg="  Apply CTE Effect", logger=self.logger)
Fang Yuedong's avatar
Fang Yuedong committed
440
441
442
            # img = effects.CTE_Effect(GSImage=img, threshold=27)
            # CTI_modeling
            # 2*8 -> 1*16 img-layout
Wei Chengliang's avatar
Wei Chengliang committed
443
444
445
446
447
448
449
450
451
452
            img = chip_utils.formatOutput(GSImage=img)
            self.nsecy = 1
            self.nsecx = 16

            img_arr = img.array
            ny, nx = img_arr.shape
            dx = int(nx/self.nsecx)
            dy = int(ny/self.nsecy)
            newimg = galsim.Image(nx, int(ny+over2), init_value=0)
            for ichannel in range(16):
Fang Yuedong's avatar
Fang Yuedong committed
453
454
455
456
457
458
459
460
461
462
463
464
                print('\n***add CTI effects: pointing-{:} chip-{:} channel-{:}***'.format(
                    pointing_ID, self.chipID, ichannel+1))
                # nx,ny,noverscan,nsp,nmax = 4608,4616,84,3,10
                noverscan, nsp, nmax = over2, 3, 10
                beta, w, c = 0.478, 84700, 0
                t = np.array([0.74, 7.7, 37], dtype=np.float32)
                rho_trap = np.array([0.6, 1.6, 1.4], dtype=np.float32)
                trap_seeds = np.array(
                    [0, 1000, 10000], dtype=np.int32) + ichannel + self.chipID*16
                release_seed = 50 + ichannel + pointing_ID*30 + self.chipID*16
                newimg.array[:, 0+ichannel*dx:dx+ichannel*dx] = CTI_sim(
                    img_arr[:, 0+ichannel*dx:dx+ichannel*dx], dx, dy, noverscan, nsp, nmax, beta, w, c, t, rho_trap, trap_seeds, release_seed)
Wei Chengliang's avatar
Wei Chengliang committed
465
466
467
468
            newimg.wcs = img.wcs
            del img
            img = newimg

Fang Yuedong's avatar
Fang Yuedong committed
469
            # 1*16 -> 2*8 img-layout
Wei Chengliang's avatar
Wei Chengliang committed
470
471
472
            img = chip_utils.formatRevert(GSImage=img)
            self.nsecy = 2
            self.nsecx = 8
473

Fang Yuedong's avatar
Fang Yuedong committed
474
        # prescan & overscan
475
        if config["ins_effects"]["add_prescan"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
476
477
            chip_utils.log_info(
                msg="  Apply pre/over-scan", logger=self.logger)
Wei Chengliang's avatar
Wei Chengliang committed
478
            if config["ins_effects"]["cte_trail"] == False:
Fang Yuedong's avatar
Fang Yuedong committed
479
480
                img = chip_utils.AddPreScan(
                    GSImage=img, pre1=pre1, pre2=pre2, over1=over1, over2=over2)
Wei Chengliang's avatar
Wei Chengliang committed
481
            if config["ins_effects"]["cte_trail"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
482
483
                img = chip_utils.AddPreScan(
                    GSImage=img, pre1=pre1, pre2=pre2, over1=over1, over2=0)
Wei Chengliang's avatar
Wei Chengliang committed
484

Fang Yuedong's avatar
Fang Yuedong committed
485
        # 1*16 output
486
        if config["ins_effects"]["format_output"] == True:
Wei Chengliang's avatar
Wei Chengliang committed
487
            chip_utils.log_info(msg="  Apply 1*16 format", logger=self.logger)
488
489
490
491
            img = chip_utils.formatOutput(GSImage=img)
            self.nsecy = 1
            self.nsecx = 16

492
493
        # Add Bias level
        if config["ins_effects"]["add_bias"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
494
495
            chip_utils.log_info(
                msg="  Adding Bias level and 16-channel non-uniformity", logger=self.logger)
496
            if config["ins_effects"]["bias_16channel"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
497
498
499
500
501
502
                img = effects.AddBiasNonUniform16(img,
                                                  bias_level=float(
                                                      self.bias_level),
                                                  nsecy=self.nsecy, nsecx=self.nsecx,
                                                  seed=SeedBiasNonuni+self.chipID,
                                                  logger=self.logger)
503
504
            elif config["ins_effects"]["bias_16channel"] == False:
                img += self.bias_level
505
506

        # Add Read-out Noise
Fang Yuedong's avatar
Fang Yuedong committed
507
        if config["ins_effects"]["add_readout"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
508
509
            seed = int(config["random_seeds"]["seed_readout"]
                       ) + pointing_ID*30 + self.chipID
510
            rng_readout = galsim.BaseDeviate(seed)
Fang Yuedong's avatar
Fang Yuedong committed
511
512
            readout_noise = galsim.GaussianNoise(
                rng=rng_readout, sigma=self.read_noise)
513
514
515
            img.addNoise(readout_noise)

        # Apply Gain & Quantization
Fang Yuedong's avatar
Fang Yuedong committed
516
517
        chip_utils.log_info(
            msg="  Applying Gain (and 16 channel non-uniformity) & Quantization", logger=self.logger)
518
        if config["ins_effects"]["gain_16channel"] == True:
519
            img, self.gain_channel = effects.ApplyGainNonUniform16(
Fang Yuedong's avatar
Fang Yuedong committed
520
521
                img, gain=self.gain,
                nsecy=self.nsecy, nsecx=self.nsecx,
522
523
524
525
                seed=SeedGainNonuni+self.chipID,
                logger=self.logger)
        elif config["ins_effects"]["gain_16channel"] == False:
            img /= self.gain
Fang Yuedong's avatar
Fang Yuedong committed
526

527
528
529
530
531
532
533
        img.array[img.array > 65535] = 65535
        img.replaceNegative(replace_value=0)
        img.quantize()

        ######################################################################################
        # Output images for calibration pointing
        ######################################################################################
Fang Yuedong's avatar
Fang Yuedong committed
534
        # Bias output
Fang Yuedong's avatar
Fang Yuedong committed
535
        if config["ins_effects"]["add_bias"] == True and config["output_setting"]["bias_output"] == True and pointing_type == 'CAL':
536
537
538
539
            if self.logger is not None:
                self.logger.info("  Output N frame Bias files")
            else:
                print("  Output N frame Bias files", flush=True)
Fang Yuedong's avatar
Fang Yuedong committed
540
            NBias = int(config["output_setting"]["NBias"])
Fang Yuedong's avatar
Fang Yuedong committed
541
            for i in range(NBias):
Fang Yuedong's avatar
Fang Yuedong committed
542
543
544
545
546
547
548
549
                # BiasCombImg, BiasTag = effects.MakeBiasNcomb(
                # self.npix_x, self.npix_y,
                # bias_level=float(self.bias_level),
                # ncombine=1, read_noise=self.read_noise, gain=1,
                # seed=SeedBiasNonuni+self.chipID,
                # logger=self.logger)
                BiasCombImg = galsim.Image(
                    self.npix_x, self.npix_y, init_value=0)
Wei Chengliang's avatar
Wei Chengliang committed
550
551
552
553
554
555
                if config["ins_effects"]["add_bias"] == True:
                    biaslevel = self.bias_level
                    overscan = biaslevel-2
                elif config["ins_effects"]["add_bias"] == False:
                    biaslevel = 0
                    overscan = 0
Wei Chengliang's avatar
Wei Chengliang committed
556

557
                # Readout noise for Biases is not generated with random seeds. So readout noise for bias images can't be reproduced.
Fang Yuedong's avatar
Fang Yuedong committed
558
559
560
                if config["ins_effects"]["cosmic_ray"] == True:
                    if config["ins_effects"]["cray_differ"] == True:
                        cr_map, cr_event_num = effects.produceCR_Map(
Fang Yuedong's avatar
Fang Yuedong committed
561
562
563
564
565
                            xLen=self.npix_x, yLen=self.npix_y,
                            exTime=0.01,
                            cr_pixelRatio=0.003 *
                            (0.01+0.5*self.readout_time)/150.,
                            gain=self.gain,
Fang Yuedong's avatar
Fang Yuedong committed
566
567
                            attachedSizes=self.attachedSizes,
                            seed=SeedCosmicRay+pointing_ID*30+self.chipID+1)
Fang Yuedong's avatar
Fang Yuedong committed
568
                        # seed: obj-imaging:+0; bias:+1; dark:+2; flat:+3;
Fang Yuedong's avatar
Fang Yuedong committed
569
570
571
                    BiasCombImg += cr_map
                    del cr_map

Fang Yuedong's avatar
Fang Yuedong committed
572
                # Apply Bad lines
Wei Chengliang's avatar
Wei Chengliang committed
573
                if config["ins_effects"]["add_badcolumns"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
574
575
                    BiasCombImg = effects.BadColumns(
                        BiasCombImg-float(self.bias_level)+5, seed=SeedBadColumns, chipid=self.chipID, logger=self.logger) + float(self.bias_level)-5
Wei Chengliang's avatar
Wei Chengliang committed
576

Fang Yuedong's avatar
Fang Yuedong committed
577
                # Non-Linearity for Bias
Fang Yuedong's avatar
Fang Yuedong committed
578
                if config["ins_effects"]["non_linear"] == True:
579
                    if self.logger is not None:
Fang Yuedong's avatar
Fang Yuedong committed
580
581
                        self.logger.info(
                            "  Applying Non-Linearity on the Bias image")
582
                    else:
Fang Yuedong's avatar
Fang Yuedong committed
583
584
585
586
587
588
589
590
591
592
593
594
                        print(
                            "  Applying Non-Linearity on the Bias image", flush=True)
                    BiasCombImg = effects.NonLinearity(
                        GSImage=BiasCombImg, beta1=5.e-7, beta2=0)

                # START
                pre1 = self.prescan_x  # 27
                over1 = self.overscan_x  # 71
                pre2 = self.prescan_y  # 0 #4
                over2 = self.overscan_y  # 84 #80

                # prescan & overscan
Wei Chengliang's avatar
Wei Chengliang committed
595
                if config["ins_effects"]["add_prescan"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
596
597
598
599
                    chip_utils.log_info(
                        msg="  Apply pre/over-scan", logger=self.logger)
                    BiasCombImg = chip_utils.AddPreScan(
                        GSImage=BiasCombImg, pre1=pre1, pre2=pre2, over1=over1, over2=over2)
Wei Chengliang's avatar
Wei Chengliang committed
600

Fang Yuedong's avatar
Fang Yuedong committed
601
                # 1*16 output
Wei Chengliang's avatar
Wei Chengliang committed
602
                if config["ins_effects"]["format_output"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
603
604
                    chip_utils.log_info(
                        msg="  Apply 1*16 format", logger=self.logger)
Wei Chengliang's avatar
Wei Chengliang committed
605
606
607
                    BiasCombImg = chip_utils.formatOutput(GSImage=BiasCombImg)
                    self.nsecy = 1
                    self.nsecx = 16
Fang Yuedong's avatar
Fang Yuedong committed
608
                # END
Wei Chengliang's avatar
Wei Chengliang committed
609

Fang Yuedong's avatar
Fang Yuedong committed
610
                # Add Bias level
Wei Chengliang's avatar
Wei Chengliang committed
611
612
                if config["ins_effects"]["add_bias"] == True:
                    if self.logger is not None:
Fang Yuedong's avatar
Fang Yuedong committed
613
614
                        self.logger.info(
                            "  Adding Bias level and 16-channel non-uniformity")
Wei Chengliang's avatar
Wei Chengliang committed
615
616
617
                    else:
                        print("  Adding Bias level and 16-channel non-uniformity")
                    BiasCombImg = effects.AddBiasNonUniform16(BiasCombImg,
Fang Yuedong's avatar
Fang Yuedong committed
618
619
620
621
                                                              bias_level=biaslevel,
                                                              nsecy=self.nsecy, nsecx=self.nsecx,
                                                              seed=SeedBiasNonuni+self.chipID,
                                                              logger=self.logger)
Wei Chengliang's avatar
Wei Chengliang committed
622
                    rng = galsim.UniformDeviate()
Fang Yuedong's avatar
Fang Yuedong committed
623
624
625
                    ncombine = 1
                    NoiseBias = galsim.GaussianNoise(
                        rng=rng, sigma=self.read_noise*ncombine**0.5)
Wei Chengliang's avatar
Wei Chengliang committed
626
                    BiasCombImg.addNoise(NoiseBias)
Fang Yuedong's avatar
Fang Yuedong committed
627

628
                BiasCombImg, self.gain_channel = effects.ApplyGainNonUniform16(BiasCombImg, gain=self.gain,
Fang Yuedong's avatar
Fang Yuedong committed
629
630
631
                                                                               nsecy=self.nsecy, nsecx=self.nsecx,
                                                                               seed=SeedGainNonuni+self.chipID,
                                                                               logger=self.logger)
Fang Yuedong's avatar
Fang Yuedong committed
632
                # BiasCombImg = effects.AddOverscan(
Fang Yuedong's avatar
Fang Yuedong committed
633
634
                #     BiasCombImg,
                #     overscan=float(config["ins_effects"]["bias_level"])-2, gain=self.gain,
Fang Yuedong's avatar
Fang Yuedong committed
635
636
637
638
                #     widthl=27, widthr=27, widtht=8, widthb=8)
                BiasCombImg.replaceNegative(replace_value=0)
                BiasCombImg.quantize()
                BiasCombImg = galsim.ImageUS(BiasCombImg)
639
                timestamp_obs += 10 * 60
640
641
                chip_utils.outputCal(
                    chip=self,
Fang Yuedong's avatar
Fang Yuedong committed
642
643
644
645
                    img=BiasCombImg,
                    ra_cen=ra_cen,
                    dec_cen=dec_cen,
                    img_rot=img_rot,
Xin Zhang's avatar
Xin Zhang committed
646
                    im_type='BIAS',
Fang Yuedong's avatar
Fang Yuedong committed
647
648
                    pointing_ID=pointing_ID,
                    output_dir=chip_output.subdir,
649
                    exptime=0.0,
650
651
                    project_cycle=config["project_cycle"],
                    run_counter=config["run_counter"],
652
                    timestamp=timestamp_obs)
Fang Yuedong's avatar
Fang Yuedong committed
653
654
655
            del BiasCombImg

        # Export combined (ncombine, Vignetting + PRNU) & single vignetting flat-field file
Fang Yuedong's avatar
Fang Yuedong committed
656
        if config["ins_effects"]["flat_fielding"] == True and config["output_setting"]["flat_output"] == True and pointing_type == 'CAL':
657
658
659
660
            if self.logger is not None:
                self.logger.info("  Output N frame Flat-Field files")
            else:
                print("  Output N frame Flat-Field files", flush=True)
Fang Yuedong's avatar
Fang Yuedong committed
661
            NFlat = int(config["output_setting"]["NFlat"])
Fang Yuedong's avatar
Fang Yuedong committed
662
            if config["ins_effects"]["add_bias"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
663
664
                biaslevel = self.bias_level
                overscan = biaslevel-2
665
            elif config["ins_effects"]["add_bias"] == False:
Fang Yuedong's avatar
Fang Yuedong committed
666
667
                biaslevel = 0
                overscan = 0
Fang Yuedong's avatar
Fang Yuedong committed
668
669
            darklevel = self.dark_noise * \
                (self.flat_exptime+0.5*self.readout_time)
Fang Yuedong's avatar
Fang Yuedong committed
670
671
672
            for i in range(NFlat):
                FlatSingle = flat_img * prnu_img + darklevel
                FlatCombImg, FlatTag = effects.MakeFlatNcomb(
Fang Yuedong's avatar
Fang Yuedong committed
673
674
                    flat_single_image=FlatSingle,
                    ncombine=1,
Fang Yuedong's avatar
Fang Yuedong committed
675
                    read_noise=self.read_noise,
Fang Yuedong's avatar
Fang Yuedong committed
676
677
                    gain=1,
                    overscan=overscan,
678
                    biaslevel=0,
679
680
                    seed_bias=SeedDefective+self.chipID,
                    logger=self.logger
Fang Yuedong's avatar
Fang Yuedong committed
681
                )
Fang Yuedong's avatar
Fang Yuedong committed
682
683
684
                if config["ins_effects"]["cosmic_ray"] == True:
                    if config["ins_effects"]["cray_differ"] == True:
                        cr_map, cr_event_num = effects.produceCR_Map(
Fang Yuedong's avatar
Fang Yuedong committed
685
686
687
688
689
                            xLen=self.npix_x, yLen=self.npix_y,
                            exTime=self.flat_exptime+0.5*self.readout_time,
                            cr_pixelRatio=0.003 *
                            (self.flat_exptime+0.5*self.readout_time)/150.,
                            gain=self.gain,
Fang Yuedong's avatar
Fang Yuedong committed
690
691
                            attachedSizes=self.attachedSizes,
                            seed=SeedCosmicRay+pointing_ID*30+self.chipID+3)
Fang Yuedong's avatar
Fang Yuedong committed
692
                        # seed: obj-imaging:+0; bias:+1; dark:+2; flat:+3;
Fang Yuedong's avatar
Fang Yuedong committed
693
                    FlatCombImg += cr_map
Fang Yuedong's avatar
Fang Yuedong committed
694
                    del cr_map
Fang Yuedong's avatar
Fang Yuedong committed
695

Wei Chengliang's avatar
Wei Chengliang committed
696
697
698
                # Add Hot Pixels or/and Dead Pixels
                rgbadpix = Generator(PCG64(int(SeedDefective+self.chipID)))
                badfraction = 5E-5*(rgbadpix.random()*0.5+0.7)
Fang Yuedong's avatar
Fang Yuedong committed
699
700
                FlatCombImg = effects.DefectivePixels(
                    FlatCombImg, IfHotPix=BoolHotPix, IfDeadPix=BoolDeadPix, fraction=badfraction, seed=SeedDefective+self.chipID, biaslevel=0)
Wei Chengliang's avatar
Wei Chengliang committed
701

Fang Yuedong's avatar
Fang Yuedong committed
702
                # Apply Bad lines
Wei Chengliang's avatar
Wei Chengliang committed
703
                if config["ins_effects"]["add_badcolumns"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
704
705
                    FlatCombImg = effects.BadColumns(
                        FlatCombImg, seed=SeedBadColumns, chipid=self.chipID, logger=self.logger)
Wei Chengliang's avatar
Wei Chengliang committed
706

Fang Yuedong's avatar
Fang Yuedong committed
707
                if config["ins_effects"]["non_linear"] == True:
708
                    if self.logger is not None:
Fang Yuedong's avatar
Fang Yuedong committed
709
710
                        self.logger.info(
                            "  Applying Non-Linearity on the Flat image")
711
                    else:
Fang Yuedong's avatar
Fang Yuedong committed
712
713
714
715
716
717
718
719
720
721
722
723
                        print(
                            "  Applying Non-Linearity on the Flat image", flush=True)
                    FlatCombImg = effects.NonLinearity(
                        GSImage=FlatCombImg, beta1=5.e-7, beta2=0)

                # if config["ins_effects"]["cte_trail"] == True:
                # FlatCombImg = effects.CTE_Effect(GSImage=FlatCombImg, threshold=3)
                # START
                pre1 = self.prescan_x  # 27
                over1 = self.overscan_x  # 71
                pre2 = self.prescan_y  # 0 #4
                over2 = self.overscan_y  # 84 #80
Fang Yuedong's avatar
Fang Yuedong committed
724
                if config["ins_effects"]["cte_trail"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
725
726
727
728
729
                    chip_utils.log_info(
                        msg="  Apply CTE Effect", logger=self.logger)
                    # img = effects.CTE_Effect(GSImage=img, threshold=27)
                    # CTI_modeling
                    # 2*8 -> 1*16 img-layout
Wei Chengliang's avatar
Wei Chengliang committed
730
731
732
733
734
735
736
737
738
739
                    FlatCombImg = chip_utils.formatOutput(GSImage=FlatCombImg)
                    self.nsecy = 1
                    self.nsecx = 16

                    img_arr = FlatCombImg.array
                    ny, nx = img_arr.shape
                    dx = int(nx/self.nsecx)
                    dy = int(ny/self.nsecy)
                    newimg = galsim.Image(nx, int(ny+over2), init_value=0)
                    for ichannel in range(16):
Fang Yuedong's avatar
Fang Yuedong committed
740
741
742
743
744
745
746
747
748
749
750
751
                        print('\n***add CTI effects: pointing-{:} chip-{:} channel-{:}***'.format(
                            pointing_ID, self.chipID, ichannel+1))
                        # nx,ny,noverscan,nsp,nmax = 4608,4616,84,3,10
                        noverscan, nsp, nmax = over2, 3, 10
                        beta, w, c = 0.478, 84700, 0
                        t = np.array([0.74, 7.7, 37], dtype=np.float32)
                        rho_trap = np.array([0.6, 1.6, 1.4], dtype=np.float32)
                        trap_seeds = np.array(
                            [0, 1000, 10000], dtype=np.int32) + ichannel + self.chipID*16
                        release_seed = 50 + ichannel + pointing_ID*30 + self.chipID*16
                        newimg.array[:, 0+ichannel*dx:dx+ichannel*dx] = CTI_sim(
                            img_arr[:, 0+ichannel*dx:dx+ichannel*dx], dx, dy, noverscan, nsp, nmax, beta, w, c, t, rho_trap, trap_seeds, release_seed)
Wei Chengliang's avatar
Wei Chengliang committed
752
753
754
755
                    newimg.wcs = FlatCombImg.wcs
                    del FlatCombImg
                    FlatCombImg = newimg

Fang Yuedong's avatar
Fang Yuedong committed
756
                    # 1*16 -> 2*8 img-layout
Wei Chengliang's avatar
Wei Chengliang committed
757
                    FlatCombImg = chip_utils.formatRevert(GSImage=FlatCombImg)
Wei Chengliang's avatar
Wei Chengliang committed
758
759
760
                    self.nsecy = 2
                    self.nsecx = 8

Fang Yuedong's avatar
Fang Yuedong committed
761
                # prescan & overscan
Wei Chengliang's avatar
Wei Chengliang committed
762
                if config["ins_effects"]["add_prescan"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
763
764
                    chip_utils.log_info(
                        msg="  Apply pre/over-scan", logger=self.logger)
Wei Chengliang's avatar
Wei Chengliang committed
765
                    if config["ins_effects"]["cte_trail"] == False:
Fang Yuedong's avatar
Fang Yuedong committed
766
767
                        FlatCombImg = chip_utils.AddPreScan(
                            GSImage=FlatCombImg, pre1=pre1, pre2=pre2, over1=over1, over2=over2)
Wei Chengliang's avatar
Wei Chengliang committed
768
                    if config["ins_effects"]["cte_trail"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
769
770
                        FlatCombImg = chip_utils.AddPreScan(
                            GSImage=FlatCombImg, pre1=pre1, pre2=pre2, over1=over1, over2=0)
Wei Chengliang's avatar
Wei Chengliang committed
771

Fang Yuedong's avatar
Fang Yuedong committed
772
                # 1*16 output
Wei Chengliang's avatar
Wei Chengliang committed
773
                if config["ins_effects"]["format_output"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
774
775
                    chip_utils.log_info(
                        msg="  Apply 1*16 format", logger=self.logger)
Wei Chengliang's avatar
Wei Chengliang committed
776
777
778
                    FlatCombImg = chip_utils.formatOutput(GSImage=FlatCombImg)
                    self.nsecy = 1
                    self.nsecx = 16
Fang Yuedong's avatar
Fang Yuedong committed
779
                # END
Fang Yuedong's avatar
Fang Yuedong committed
780

781
                # Add Bias level
Fang Yuedong's avatar
Fang Yuedong committed
782
                if config["ins_effects"]["add_bias"] == True:
783
                    if self.logger is not None:
Fang Yuedong's avatar
Fang Yuedong committed
784
785
                        self.logger.info(
                            "  Adding Bias level and 16-channel non-uniformity")
786
787
                    else:
                        print("  Adding Bias level and 16-channel non-uniformity")
Fang Yuedong's avatar
Fang Yuedong committed
788
                    # img += float(config["ins_effects"]["bias_level"])
Fang Yuedong's avatar
Fang Yuedong committed
789
790
791
792
793
794
                    FlatCombImg = effects.AddBiasNonUniform16(FlatCombImg,
                                                              bias_level=biaslevel,
                                                              nsecy=self.nsecy, nsecx=self.nsecx,
                                                              seed=SeedBiasNonuni+self.chipID,
                                                              logger=self.logger)

795
                # Add Read-out Noise
Fang Yuedong's avatar
Fang Yuedong committed
796
                if config["ins_effects"]["add_readout"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
797
798
                    seed = int(config["random_seeds"]["seed_readout"]
                               ) + pointing_ID*30 + self.chipID + 3
799
                    rng_readout = galsim.BaseDeviate(seed)
Fang Yuedong's avatar
Fang Yuedong committed
800
801
                    readout_noise = galsim.GaussianNoise(
                        rng=rng_readout, sigma=self.read_noise)
802
                    FlatCombImg.addNoise(readout_noise)
Fang Yuedong's avatar
Fang Yuedong committed
803

804
                FlatCombImg, self.gain_channel = effects.ApplyGainNonUniform16(FlatCombImg, gain=self.gain,
Fang Yuedong's avatar
Fang Yuedong committed
805
806
807
                                                                               nsecy=self.nsecy, nsecx=self.nsecx,
                                                                               seed=SeedGainNonuni+self.chipID,
                                                                               logger=self.logger)
Fang Yuedong's avatar
Fang Yuedong committed
808
809
810
811
                # FlatCombImg = effects.AddOverscan(FlatCombImg, overscan=overscan, gain=self.gain, widthl=27, widthr=27, widtht=8, widthb=8)
                FlatCombImg.replaceNegative(replace_value=0)
                FlatCombImg.quantize()
                FlatCombImg = galsim.ImageUS(FlatCombImg)
812
                timestamp_obs += 10 * 60
813
814
                chip_utils.outputCal(
                    chip=self,
Fang Yuedong's avatar
Fang Yuedong committed
815
816
817
818
                    img=FlatCombImg,
                    ra_cen=ra_cen,
                    dec_cen=dec_cen,
                    img_rot=img_rot,
Xin Zhang's avatar
Xin Zhang committed
819
                    im_type='FLAT',
Fang Yuedong's avatar
Fang Yuedong committed
820
821
                    pointing_ID=pointing_ID,
                    output_dir=chip_output.subdir,
822
                    exptime=self.flat_exptime,
823
824
                    project_cycle=config["project_cycle"],
                    run_counter=config["run_counter"],
825
                    timestamp=timestamp_obs)
Fang Yuedong's avatar
Fang Yuedong committed
826

Fang Yuedong's avatar
Fang Yuedong committed
827
828
829
830
831
832
833
            del FlatCombImg, FlatSingle, prnu_img
            # flat_img.replaceNegative(replace_value=0)
            # flat_img.quantize()
            # galsim.ImageUS(flat_img).write("%s/FlatImg_Vignette_%s.fits" % (chip_output.subdir, self.chipID))
            del flat_img

        # Export Dark current images
Fang Yuedong's avatar
Fang Yuedong committed
834
        if config["ins_effects"]["add_dark"] == True and config["output_setting"]["dark_output"] == True and pointing_type == 'CAL':
835
836
837
838
            if self.logger is not None:
                self.logger.info("  Output N frame Dark Current files")
            else:
                print("  Output N frame Dark Current files", flush=True)
Fang Yuedong's avatar
Fang Yuedong committed
839
            NDark = int(config["output_setting"]["NDark"])
Fang Yuedong's avatar
Fang Yuedong committed
840
            if config["ins_effects"]["add_bias"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
841
842
                biaslevel = self.bias_level
                overscan = biaslevel-2
843
            elif config["ins_effects"]["add_bias"] == False:
Fang Yuedong's avatar
Fang Yuedong committed
844
845
846
847
                biaslevel = 0
                overscan = 0
            for i in range(NDark):
                DarkCombImg, DarkTag = effects.MakeDarkNcomb(
Fang Yuedong's avatar
Fang Yuedong committed
848
                    self.npix_x, self.npix_y,
849
                    overscan=overscan, bias_level=0, darkpsec=0.02, exptime=self.dark_exptime+0.5*self.readout_time,
Fang Yuedong's avatar
Fang Yuedong committed
850
                    ncombine=1, read_noise=self.read_noise,
851
852
                    gain=1, seed_bias=SeedBiasNonuni+self.chipID,
                    logger=self.logger)
Fang Yuedong's avatar
Fang Yuedong committed
853
854
855
                if config["ins_effects"]["cosmic_ray"] == True:
                    if config["ins_effects"]["cray_differ"] == True:
                        cr_map, cr_event_num = effects.produceCR_Map(
Fang Yuedong's avatar
Fang Yuedong committed
856
857
858
859
860
                            xLen=self.npix_x, yLen=self.npix_y,
                            exTime=self.dark_exptime+0.5*self.readout_time,
                            cr_pixelRatio=0.003 *
                            (self.dark_exptime+0.5*self.readout_time)/150.,
                            gain=self.gain,
Fang Yuedong's avatar
Fang Yuedong committed
861
862
                            attachedSizes=self.attachedSizes,
                            seed=SeedCosmicRay+pointing_ID*30+self.chipID+2)
Fang Yuedong's avatar
Fang Yuedong committed
863
                        # seed: obj-imaging:+0; bias:+1; dark:+2; flat:+3;
Fang Yuedong's avatar
Fang Yuedong committed
864
                    DarkCombImg += cr_map
Fang Yuedong's avatar
Fang Yuedong committed
865
866
867
868
                    cr_map[cr_map > 65535] = 65535
                    cr_map[cr_map < 0] = 0
                    crmap_gsimg = galsim.Image(cr_map, dtype=np.uint16)
                    del cr_map
Fang Yuedong's avatar
Fang Yuedong committed
869
870
                    # START
                    # prescan & overscan
Wei Chengliang's avatar
Wei Chengliang committed
871
                    if config["ins_effects"]["add_prescan"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
872
873
874
875
                        chip_utils.log_info(
                            msg="  Apply pre/over-scan", logger=self.logger)
                        crmap_gsimg = chip_utils.AddPreScan(
                            GSImage=crmap_gsimg, pre1=pre1, pre2=pre2, over1=over1, over2=over2)
Wei Chengliang's avatar
Wei Chengliang committed
876

Fang Yuedong's avatar
Fang Yuedong committed
877
                    # 1*16 output
Wei Chengliang's avatar
Wei Chengliang committed
878
                    if config["ins_effects"]["format_output"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
879
880
881
882
                        chip_utils.log_info(
                            msg="  Apply 1*16 format", logger=self.logger)
                        crmap_gsimg = chip_utils.formatOutput(
                            GSImage=crmap_gsimg)
Wei Chengliang's avatar
Wei Chengliang committed
883
884
                        self.nsecy = 1
                        self.nsecx = 16
Fang Yuedong's avatar
Fang Yuedong committed
885
                    # END
886
887
                    chip_utils.outputCal(
                        chip=self,
Fang Yuedong's avatar
Fang Yuedong committed
888
889
890
891
892
893
894
                        img=crmap_gsimg,
                        ra_cen=ra_cen,
                        dec_cen=dec_cen,
                        img_rot=img_rot,
                        im_type='CRD',
                        pointing_ID=pointing_ID,
                        output_dir=chip_output.subdir,
895
                        exptime=self.dark_exptime,
896
897
                        project_cycle=config["project_cycle"],
                        run_counter=config["run_counter"],
898
                        timestamp=timestamp_obs)
Fang Yuedong's avatar
Fang Yuedong committed
899
                    del crmap_gsimg
Fang Yuedong's avatar
Fang Yuedong committed
900

Wei Chengliang's avatar
Wei Chengliang committed
901
902
903
                # Add Hot Pixels or/and Dead Pixels
                rgbadpix = Generator(PCG64(int(SeedDefective+self.chipID)))
                badfraction = 5E-5*(rgbadpix.random()*0.5+0.7)
Fang Yuedong's avatar
Fang Yuedong committed
904
905
                DarkCombImg = effects.DefectivePixels(
                    DarkCombImg, IfHotPix=BoolHotPix, IfDeadPix=BoolDeadPix, fraction=badfraction, seed=SeedDefective+self.chipID, biaslevel=0)
Wei Chengliang's avatar
Wei Chengliang committed
906

Fang Yuedong's avatar
Fang Yuedong committed
907
                # Apply Bad lines
Wei Chengliang's avatar
Wei Chengliang committed
908
                if config["ins_effects"]["add_badcolumns"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
909
910
                    DarkCombImg = effects.BadColumns(
                        DarkCombImg, seed=SeedBadColumns, chipid=self.chipID, logger=self.logger)
Wei Chengliang's avatar
Wei Chengliang committed
911

Fang Yuedong's avatar
Fang Yuedong committed
912
                # Non-Linearity for Dark
Fang Yuedong's avatar
Fang Yuedong committed
913
                if config["ins_effects"]["non_linear"] == True:
914
                    if self.logger is not None:
Fang Yuedong's avatar
Fang Yuedong committed
915
916
                        self.logger.info(
                            "  Applying Non-Linearity on the Dark image")
917
                    else:
Fang Yuedong's avatar
Fang Yuedong committed
918
919
920
921
922
923
924
925
926
927
928
929
                        print(
                            "  Applying Non-Linearity on the Dark image", flush=True)
                    DarkCombImg = effects.NonLinearity(
                        GSImage=DarkCombImg, beta1=5.e-7, beta2=0)

                # if config["ins_effects"]["cte_trail"] == True:
                # DarkCombImg = effects.CTE_Effect(GSImage=DarkCombImg, threshold=3)
                # START
                pre1 = self.prescan_x  # 27
                over1 = self.overscan_x  # 71
                pre2 = self.prescan_y  # 0 #4
                over2 = self.overscan_y  # 84 #80
Fang Yuedong's avatar
Fang Yuedong committed
930
                if config["ins_effects"]["cte_trail"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
931
932
933
934
935
                    chip_utils.log_info(
                        msg="  Apply CTE Effect", logger=self.logger)
                    # img = effects.CTE_Effect(GSImage=img, threshold=27)
                    # CTI_modeling
                    # 2*8 -> 1*16 img-layout
Wei Chengliang's avatar
Wei Chengliang committed
936
937
938
939
940
941
942
943
944
945
                    DarkCombImg = chip_utils.formatOutput(GSImage=DarkCombImg)
                    self.nsecy = 1
                    self.nsecx = 16

                    img_arr = DarkCombImg.array
                    ny, nx = img_arr.shape
                    dx = int(nx/self.nsecx)
                    dy = int(ny/self.nsecy)
                    newimg = galsim.Image(nx, int(ny+over2), init_value=0)
                    for ichannel in range(16):
Fang Yuedong's avatar
Fang Yuedong committed
946
947
948
949
950
951
952
953
954
955
956
957
                        print('\n***add CTI effects: pointing-{:} chip-{:} channel-{:}***'.format(
                            pointing_ID, self.chipID, ichannel+1))
                        # nx,ny,noverscan,nsp,nmax = 4608,4616,84,3,10
                        noverscan, nsp, nmax = over2, 3, 10
                        beta, w, c = 0.478, 84700, 0
                        t = np.array([0.74, 7.7, 37], dtype=np.float32)
                        rho_trap = np.array([0.6, 1.6, 1.4], dtype=np.float32)
                        trap_seeds = np.array(
                            [0, 1000, 10000], dtype=np.int32) + ichannel + self.chipID*16
                        release_seed = 50 + ichannel + pointing_ID*30 + self.chipID*16
                        newimg.array[:, 0+ichannel*dx:dx+ichannel*dx] = CTI_sim(
                            img_arr[:, 0+ichannel*dx:dx+ichannel*dx], dx, dy, noverscan, nsp, nmax, beta, w, c, t, rho_trap, trap_seeds, release_seed)
Wei Chengliang's avatar
Wei Chengliang committed
958
959
960
961
                    newimg.wcs = DarkCombImg.wcs
                    del DarkCombImg
                    DarkCombImg = newimg

Fang Yuedong's avatar
Fang Yuedong committed
962
                    # 1*16 -> 2*8 img-layout
Wei Chengliang's avatar
Wei Chengliang committed
963
964
965
966
                    DarkCombImg = chip_utils.formatRevert(GSImage=DarkCombImg)
                    self.nsecy = 2
                    self.nsecx = 8

Fang Yuedong's avatar
Fang Yuedong committed
967
                # prescan & overscan
Wei Chengliang's avatar
Wei Chengliang committed
968
                if config["ins_effects"]["add_prescan"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
969
970
                    chip_utils.log_info(
                        msg="  Apply pre/over-scan", logger=self.logger)
Wei Chengliang's avatar
Wei Chengliang committed
971
                    if config["ins_effects"]["cte_trail"] == False:
Fang Yuedong's avatar
Fang Yuedong committed
972
973
                        DarkCombImg = chip_utils.AddPreScan(
                            GSImage=DarkCombImg, pre1=pre1, pre2=pre2, over1=over1, over2=over2)
Wei Chengliang's avatar
Wei Chengliang committed
974
                    if config["ins_effects"]["cte_trail"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
975
976
                        DarkCombImg = chip_utils.AddPreScan(
                            GSImage=DarkCombImg, pre1=pre1, pre2=pre2, over1=over1, over2=0)
Wei Chengliang's avatar
Wei Chengliang committed
977

Fang Yuedong's avatar
Fang Yuedong committed
978
                # 1*16 output
Wei Chengliang's avatar
Wei Chengliang committed
979
                if config["ins_effects"]["format_output"] == True:
Fang Yuedong's avatar
Fang Yuedong committed
980
981
                    chip_utils.log_info(
                        msg="  Apply 1*16 format", logger=self.logger)
Wei Chengliang's avatar
Wei Chengliang committed
982
983
984
                    DarkCombImg = chip_utils.formatOutput(GSImage=DarkCombImg)
                    self.nsecy = 1
                    self.nsecx = 16
Fang Yuedong's avatar
Fang Yuedong committed
985
                # END
Fang Yuedong's avatar
Fang Yuedong committed
986

987
                # Add Bias level
Fang Yuedong's avatar
Fang Yuedong committed
988
                if config["ins_effects"]["add_bias"] == True:
989
                    if self.logger is not None:
Fang Yuedong's avatar
Fang Yuedong committed
990
991
                        self.logger.info(
                            "  Adding Bias level and 16-channel non-uniformity")
992
993
                    else:
                        print("  Adding Bias level and 16-channel non-uniformity")
Fang Yuedong's avatar
Fang Yuedong committed
994
                    # img += float(config["ins_effects"]["bias_level"])
Fang Yuedong's avatar
Fang Yuedong committed
995
996
997
998
999
                    DarkCombImg = effects.AddBiasNonUniform16(DarkCombImg,
                                                              bias_level=biaslevel,
                                                              nsecy=self.nsecy, nsecx=self.nsecx,
                                                              seed=SeedBiasNonuni+self.chipID,
                                                              logger=self.logger)
1000

For faster browsing, not all history is shown. View entire blame