first commit

Grid.py

+import math
+import numpy as np
+import matplotlib.pyplot as plt
+
+class BaseGrid(object):
+    _valid_grid_types = ['RectGrid', 'HexGrid']
+    _valid_mixed_types = ['MixedGrid']
+
+class Grid(BaseGrid):
+    def __init__(self, grid_spacing, wcs, Npix_x=10000, Npix_y=10000, pixelscale=0.074, rot_angle=None, pos_offset=None, angle_unit='rad'):
+        self.grid_spacing = grid_spacing
+        self.im_gs = grid_spacing * (1.0 / pixelscale) # pixels
+        self.pixelscale = pixelscale
+        self.Npix_x, self.Npix_y = Npix_x, Npix_y
+        self.wcs = wcs
+        self.rot_angle = rot_angle # rotation angle, in rad
+        self.angle_unit = angle_unit
+        if pos_offset:
+            self.pos_offset = np.array(pos_offset)
+        else:
+            self.pos_offset = np.array([0., 0.])
+
+        # May have to modify grid corners if there is a rotation
+        if rot_angle:
+            dx = Npix_x / 2.
+            dy = Npix_y / 2.
+            if angle_unit == 'deg':
+                theta = np.deg2rad(rot_angle)
+            else:
+                theta = rot_angle
+            self.startx = (0.-dx) * np.cos(theta) - (Npix_y-dy) * np.sin(theta) + dx
+            self.endx = (Npix_x-dx) * np.cos(theta) - (0.-dy) * np.sin(theta) + dx
+            self.starty = (0.-dx) * np.cos(theta) + (0.-dy) * np.sin(theta) + dx
+            self.endy = (Npix_x-dx) * np.cos(theta) + (Npix_y-dy) * np.sin(theta) + dx
+        else:
+            self.startx, self.endx= 0., Npix_x
+            self.starty, self.endy= 0., Npix_y
+
+    def rotate_grid(self, theta, offset=None, angle_unit='rad'):
+
+        if angle_unit == 'deg':
+            theta = np.deg2rad(theta)
+        elif angle_unit != 'rad':
+            raise ValueError('`angle_unit` can only be `deg` or `rad`! ' +
+                                  'Passed unit of {}'.format(angle_unit))
+
+        if not offset: offset = [0., 0.]
+
+        c, s = np.cos(theta), np.sin(theta)
+        R = np.array(((c,-s), (s, c)))
+
+        offset_grid = np.array([self.im_ra - offset[0], self.im_dec - offset[1]])
+        translate = np.empty_like(offset_grid)
+        translate[0,:] = offset[0]
+        translate[1,:] = offset[1]
+
+        rotated_grid = np.dot(R, offset_grid) + translate
+
+        self.im_pos = rotated_grid.T
+        self.im_ra, self.im_dec = self.im_pos[0,:], self.im_pos[1,:]
+
+    def cut2buffer(self):
+        '''
+        Remove objects outside of tile (and buffer).
+        We must sample points in the buffer zone in the beginning due to
+        possible rotations.
+        '''
+        b = self.im_gs
+        in_region = np.where( (self.im_pos[:,0]>b) & (self.im_pos[:,0]<self.Npix_x-b) &
+                              (self.im_pos[:,1]>b) & (self.im_pos[:,1]<self.Npix_y-b) )
+        self.im_pos = self.im_pos[in_region]
+        self.im_ra = self.im_pos[:,0]
+        self.im_dec = self.im_pos[:,1]
+
+        # Get all image coordinate pairs
+        self.pos = self.wcs.wcs_pix2world(self.im_pos, 1)
+        self.ra = self.pos[:,0]
+        self.dec = self.pos[:,1]
+
+class RectGrid(Grid):
+    def __init__(self, grid_spacing, wcs, Npix_x=10000, Npix_y=10000, pixelscale=0.074,
+                 rot_angle=None, pos_offset=None, angle_unit='rad'):
+        super(RectGrid, self).__init__(grid_spacing, wcs, Npix_x=Npix_x, Npix_y=Npix_y,
+                                       pixelscale=pixelscale, rot_angle=rot_angle,
+                                       pos_offset=pos_offset, angle_unit=angle_unit)
+        self._create_grid()
+    
+    def _create_grid(self):
+        im_gs = self.im_gs
+
+        po = self.pos_offset
+        im_po = po / self.pixelscale
+        self.im_ra  = np.arange(self.startx, self.endx, im_gs)
+        self.im_dec = np.arange(self.starty, self.endy, im_gs)
+
+        # Get all image coordinate pairs
+        self.im_pos = np.array(np.meshgrid(self.im_ra, self.im_dec)).T.reshape(
+                -1, 2)
+        self.im_ra  = self.im_pos[:,0]
+        self.im_dec = self.im_pos[:,1]
+
+        if self.rot_angle:
+            self.rotate_grid(self.rot_angle, angle_unit=self.angle_unit,
+                            offset=[(self.Npix_x+im_po[0])/2., (self.Npix_y+im_po[1])/2.])
+
+        self.cut2buffer()
+
+class HexGrid(Grid):
+    def __init__(self, grid_spacing, wcs, Npix_x=10000, Npix_y=10000, pixelscale=0.074,
+                 rot_angle=None, pos_offset=None, angle_unit='rad'):
+        super(HexGrid, self).__init__(grid_spacing, wcs, Npix_x=Npix_x, Npix_y=Npix_y,
+                                       pixelscale=pixelscale, rot_angle=rot_angle,
+                                       pos_offset=pos_offset, angle_unit=angle_unit)
+        self._create_grid()
+
+    def _create_grid(self):
+        im_gs = self.im_gs
+
+        po = self.pos_offset
+        im_po = [p / self.pixelscale for p in po]
+        self.im_pos = HexGrid.calc_hex_coords(self.startx, self.starty, self.endx, self.endy, im_gs)
+        self.im_ra = self.im_pos[:, 0]
+        self.im_dec = self.im_pos[:, 1]
+
+        if self.rot_angle:
+            self.rotate_grid(self.rot_angle, angle_unit=self.angle_unit,
+                            offset=[(self.Npix_x+im_po[0])/2., (self.Npix_y+im_po[1])/2.])
+        self.cut2buffer()
+
+    @classmethod
+    def calc_hex_coords(cls, startx, starty, endx, endy, radius):
+        # Geoemtric factors of given hexagon
+        r = radius
+        p = r * np.tan(np.pi / 6.) # side length / 2
+        h = 4. * p
+        dx = 2. * r
+        dy = 2. * p
+
+        row = 1
+
+        xs = []
+        ys = []
+
+        while startx < endx:
+            x = [startx, startx, startx + r, startx + dx, startx + dx, startx + r, startx + r]
+            xs.append(x)
+            startx += dx
+
+        while starty < endy:
+            y = [starty + p, starty + 3*p, starty + h, starty + 3*p, starty + p, starty, starty + dy]
+            ys.append(y)
+            starty += 2*p
+            row += 1
+        
+        print(xs)
+        print(ys)
+
+        polygons = [zip(x, y) for x in xs for y in ys]
+        polygons = [np.column_stack((x, y)) for x in xs for y in ys]
+        # polygons = np.array(polygons)
+        hexgrid = cls.polygons2coords(polygons)
+
+        # Some hexagonal elements go beyond boundary; cut these out
+        indx = np.where( (hexgrid[:,0]<endx) & (hexgrid[:,1]<endy) )
+        return hexgrid[indx]
+
+    @classmethod
+    def polygons2coords(HexGrid, p):
+        print(p)
+        s = np.shape(p)
+        print(s)
+        L = s[0]*s[1]
+        pp = np.array(p).reshape(L,2)
+        c = np.vstack({tuple(row) for row in pp})
+        # Some of the redundant coordinates are offset by ~1e-10 pixels
+        return np.unique(c.round(decimals=6), axis=0)
+
+def _build_grid(grid_type, **kwargs):
+    if grid_type in GRID_TYPES:
+        return GRID_TYPES[grid_type](**kwargs)
+    else:
+        raise ValueError('There is not yet an implemnted default Grid of type {}'.format(grid_type))
+
+GRID_TYPES = {
+    'RectGrid': RectGrid,
+    'HexGrid': HexGrid
+}
\ No newline at end of file

InjectionCatalog.py

+import numpy as np
+import galsim
+
+import Grid
+import mathutil as util
+
+class InjectionCatalog(object):
+
+    def __init__(self, image):
+        self.rmin, self.rmax = image.ramin, image.ramax
+        self.decmin, self.decmax = image.decmin, image.decmax
+        self.ra_boundary_cross = image.ra_boundary_cross
+        self.Npix_x, self.Npix_y = image.Npix_x, image.Npix_y
+        self.pixel_scale = image.pixel_scale
+        self.wcs = image.wcs
+
+        # This is also set for a given image, as it may depend on its area
+        self.objs_per_real = image.objs_per_real
+
+        # TODO: output truth catalog after injection
+        self.truth_outfile = {}
+        pass
+
+    def generate_positions(self, config):
+        
+        bg = Grid.BaseGrid()
+
+        self.nobjects = self.objs_per_real
+        ps = config['pos_sampling']
+        pstype = ps['type']
+        
+        if pstype == 'uniform' and (self.objs_per_real is not None):
+            ra = util.sample_uniform_ra(self.ramin, self.ramax, self.objs_per_real, 
+                                        boundary_cross=self.ra_boundary_cross)
+            dec = util.sample_uniform_dec(self.decmin, self.decmax, self.objs_per_real, unit='deg')
+            self.pos = np.column_stack((ra, dec))
+        
+        elif pstype in bg._valid_grid_types:
+            grid_kwargs = self._build_grid_kwargs(pstype, ps)
+            gtype = pstype
+            
+            img_grid = Grid._build_grid(gtype, **grid_kwargs)
+            self.pos = img_grid.pos
+            self.nobjects = np.shape(img_grid.pos)[0]
+        
+        elif pstype in bg._valid_mixed_types:
+            pass
+        
+        else:
+            raise ValueError('Position sampling type {} is not valid!'.format(gtype))
+
+        # Generate object indices (in input catalog)
+        # TODO
+        # self.index = np.random.choice(xrange(input_nobjects), size=self.nobjects)
+
+        # Generate object rotation angles, if desired
+        if config['rotate_objs'] is True:
+            rot = util.sample_uniform(0., 360., self.nobjects)
+            self.rotate = np.array([str(r) + ' deg' for r in rot])
+        else:
+            self.rotate = None
+
+
+    def _build_grid_kwargs(self, pstype, ps):
+        if pstype == 'MixedGrid':
+            gtype = ps['grid_type']
+        else:
+            gtype = pstype
+
+        gs = ps['grid_spacing']
+        
+        try:
+            r = ps['rotate']
+            if (isinstance(r, str) and (r.lower() == 'random')):
+                if gtype == 'RectGrid':
+                    self.grid_rot_angle = np.random.uniform(0., np.pi/2.)
+                elif gtype == 'HexGrid':
+                    self.grid_rot_angle = np.random.uniform(0., np.pi/3.)
+            else:
+                unit = ps['angle_unit']
+                if unit == 'deg':
+                    if (r >= 0.) and (r < 360.):
+                        self.grid_rot_angle = float(r)
+                    else:
+                        raise ValueError('Grid rotation of {} '.format(r) + 'deg is not valid!')
+                else:
+                    if (r >= 0.) and (r < 2*np.pi):
+                        self.grid_rot_angle = float(r)
+                    else:
+                        raise ValueError('Grid rotation of {} '.format(r) + 'rad is not valid!')
+        except KeyError:
+            self.grid_rot_angle = 0.0
+
+        # Offset grid if desired
+        try:
+            offset = ps['offset']
+            if (isinstance(offset, str)) and (offset.lower() == 'random'):
+                self.grid_offset = [np.random.uniform(-gs/2., gs/2.),
+                                    np.random.uniform(-gs/2., gs/2.)]
+            else:
+                if isinstance(offset, list):
+                    self.grid_offset = list(offset)
+                else:
+                    raise ValueError('Grid offset of {} '.format(offset) + 'is not valid!')
+        except KeyError:
+            self.grid_offset = [0., 0.]
+
+        try:
+            self.angle_unit = ps['angle_unit']
+        except KeyError:
+            self.angle_unit = 'rad'
+
+        grid_kwargs = dict(grid_spacing=gs,
+                        wcs=self.wcs,
+                        Npix_x=self.Npix_x,
+                        Npix_y=self.Npix_y,
+                        pixelscale=self.pixel_scale,
+                        rot_angle=self.grid_rot_angle,
+                        angle_unit=self.angle_unit,
+                        pos_offset=self.grid_offset)
+        return grid_kwargs
+
+    def get_truth_outfile(self):
+        pass

Injector.py

+import galsim
+import galsim.config.stamp as stamp
+import logging
+import os
+import numpy as np
+
+import grid
+
+class AddOnImageBuilder(galsim.config.image_scattered.ScatteredImageBuilder):
+    
+    def setup(self, config, base, image_num, obj_num, ignore, logger):
+        ignore = ignore + ['initial_image']
+        return super(AddOnImageBuilder, self).setup(config, base, image_num, obj_num, ignore, logger)
+    
+    def addNoise(self, image, config, base, image_num, obj_num, current_var, logger):
+        super(AddOnImageBuilder, self).addNoise(image, config, base, image_num, obj_num, current_var, logger)
+
+        initial_image_name = galsim.config.ParseValue(config, 'initial_image', base, str)[0]
+        initial_image = galsim.fits.read(initial_image_name)
+        image += initial_image
+
+
+galsim.config.RegisterImageType('AddOnImage', AddOnImageBuilder())
+
+class InjectImageBuilder(AddOnImageBuilder):
+
+    def setup(self, config, base, image_num, obj_num, ignore, logger):
+        extra_ignore = ignore + ['tile_list', 'geom_file', 'tile_dir', 'config_dir', 'psf_dir',
+                                 'version', 'run_name', 'bands', 'n_objects', 'n_realizations',
+                                 'object_density', 'inj_objs_only', 'pos_sampling', 'realizations',
+                                 'extinct_objs', 'rotate_objs']
+        for key in config:
+            if 'N_' in key:
+                extra_ignore.append(key)
+        full_xsize, full_ysize = super(InjectImageBuilder, self).setup(config, base, image_num, obj_num, extra_ignore, logger)
+
+        # config = parse_inject_image_inputs(config, base)
+
+        return full_xsize, full_ysize
+
+    def addNoise(self, image, config, base, image_num, obj_num, current_var, logger):
+        try:
+            ioo = config['inj_objs_only']
+            if (type(ioo) is bool) and (ioo is True):
+                return super(AddOnImageBuilder, self).addNoise(image,
+                                                               config,
+                                                               base,
+                                                               image_num,
+                                                               obj_num,
+                                                               current_var,
+                                                               logger)
+            elif (isinstance(ioo, dict)) and (ioo['value'] is True):
+                # Still want to use existing image if changed to be BKG
+                if (ioo['noise']) and ('BKG' in ioo['noise']):
+                    return super(InjectImageBuilder, self).addNoise(image,
+                                                                    config,
+                                                                    base,
+                                                                    image_num,
+                                                                    obj_num,
+                                                                    current_var,
+                                                                    logger)
+                else:
+                    return super(AddOnImageBuilder, self).addNoise(image,
+                                                                   config,
+                                                                   base,
+                                                                   image_num,
+                                                                   obj_num,
+                                                                   current_var,
+                                                                   logger)
+            else:
+                # Default is to add on top of initial images
+                return super(InjectImageBuilder, self).addNoise(image,
+                                                                config,
+                                                                base,
+                                                                image_num,
+                                                                obj_num,
+                                                                current_var,
+                                                                logger)
+
+        except KeyError:
+            # Default is to add on top of initial images
+            return super(InjectImageBuilder, self).addNoise(image,
+                                                            config,
+                                                            base,
+                                                            image_num,
+                                                            obj_num,
+                                                            current_var,
+                                                            logger)
+
+galsim.config.RegisterImageType('InjectImage', InjectImageBuilder())

InputCatalogs.py

+import os
+import numpy as np
+import h5py as h5
+import random
+import galsim
+import astropy.constants as cons
+from astropy.table import Table
+from scipy import interpolate
+
+from ObservationSim.MockObject import CatalogBase, Star, Galaxy, Quasar
+from ObservationSim.MockObject._util import seds, sed_assign, extAv, tag_sed, getObservedSED
+
+try:
+    import importlib.resources as pkg_resources
+except ImportError:
+    # Try backported to PY<37 'importlib_resources'
+    import importlib_resources as pkg_resources
+
+class SimCat(CatalogBase):
+    def __init__(self, config, chip, nobjects=None):
+        super().__init__()
+        self.cat_dir = os.path.join(config["data_dir"], config["input_path"]["cat_dir"])
+        self.config = config
+        self.chip = chip
+        self.seed_Av = config["random_seeds"]["seed_Av"]
+
+        with pkg_resources.path('Catalog.data', 'SLOAN_SDSS.g.fits') as filter_path:
+            self.normF_star = Table.read(str(filter_path))
+        with pkg_resources.path('Catalog.data', 'lsst_throuput_g.fits') as filter_path:
+            self.normF_galaxy = Table.read(str(filter_path))
+        if "star_cat" in config["input_path"] and config["input_path"]["star_cat"]:
+            star_file = config["input_path"]["star_cat"]
+            star_SED_file = config["SED_templates_path"]["star_SED"]
+            self.star_path = os.path.join(self.cat_dir, star_file)
+            self.star_SED_path = os.path.join(config["data_dir"], star_SED_file)
+            self._load_SED_lib_star()
+        if "galaxy_cat" in config["input_path"] and config["input_path"]["galaxy_cat"]:
+            galaxy_file = config["input_path"]["galaxy_cat"]
+            self.galaxy_path = os.path.join(self.cat_dir, galaxy_file)
+            self.galaxy_SED_path = os.path.join(config["data_dir"], config["SED_templates_path"]["galaxy_SED"])
+            self._load_SED_lib_gals()
+
+        self._load(nobjects=nobjects)
+    
+    def _load_SED_lib_star(self):
+        self.tempSED_star = h5.File(self.star_SED_path,'r')
+
+    def _load_SED_lib_gals(self):
+        self.tempSed_gal, self.tempRed_gal = seds("galaxy.list", seddir=self.galaxy_SED_path)
+
+    def load_norm_filt(self, obj):
+        if obj.type == "star":
+            return self.normF_star
+        elif obj.type == "galaxy" or obj.type == "quasar":
+            return self.normF_galaxy
+        else:
+            return None
+
+    def _load_gals(self, gals, pix_id=None, nobjects=None):
+        # Load how mnay objects?
+        if nobjects is None:
+            ngals = 5000
+        else:
+            ngals = nobjects
+        self.rng_sedGal = random.Random()
+        self.rng_sedGal.seed(pix_id) # Use healpix index as the random seed
+        self.ud = galsim.UniformDeviate(pix_id)
+
+        for igals in range(ngals):
+            param = self.initialize_param()
+            param['ra'] = gals['ra_true'][igals]
+            param['dec'] = gals['dec_true'][igals]
+
+            # param['mag_use_normal'] = gals['mag_true_g_lsst'][igals]
+            # (TEST) use same magnitude 
+            # (there will be slight difference due to randomness in SED)
+            param['mag_use_normal'] = 18
+            
+            param['z'] = gals['redshift_true'][igals]
+            param['model_tag'] = 'None'
+            param['gamma1'] = 0
+            param['gamma2'] = 0
+            param['kappa'] = 0
+            param['delta_ra'] = 0
+            param['delta_dec'] = 0
+
+            hlrMajB = gals['size_bulge_true'][igals]
+            hlrMinB = gals['size_minor_bulge_true'][igals]
+
+            hlrMajD = gals['size_disk_true'][igals]
+            hlrMinD = gals['size_minor_disk_true'][igals]
+            aGal = gals['size_true'][igals]
+            bGal = gals['size_minor_true'][igals]
+            param['bfrac'] = gals['bulge_to_total_ratio_i'][igals]
+            param['theta'] = gals['position_angle_true'][igals]
+            param['hlr_bulge'] = np.sqrt(hlrMajB * hlrMinB)
+            param['hlr_disk'] = np.sqrt(hlrMajD * hlrMinD)
+            param['ell_bulge'] = (hlrMajB - hlrMinB)/(hlrMajB + hlrMinB)
+            param['ell_disk'] = (hlrMajD - hlrMinD)/(hlrMajD + hlrMinD)
+            param['ell_tot'] = (aGal - bGal) / (aGal + bGal)
+
+            # Assign each galaxy a template SED
+            param['sed_type'] = sed_assign(phz=param['z'], btt=param['bfrac'], rng=self.rng_sedGal)
+            param['redden'] = self.tempRed_gal[param['sed_type']]
+            param['av'] = self.avGal[int(self.ud()*self.nav)]
+            if param['sed_type'] <= 5:
+                param['av'] = 0.0
+                param['redden'] = 0
+            param['star'] = 0   # Galaxy
+            if param['sed_type'] >= 29:
+                param['av'] = 0.6 * param['av'] / 3.0 # for quasar, av=[0, 0.2], 3.0=av.max-av.im
+                param['star'] = 2 # Quasar
+            
+            param['id'] = gals['galaxyID'][igals]
+            
+            if param['star'] == 0:
+                obj = Galaxy(param)
+            if param['star'] == 2:
+                obj = Quasar(param)
+            
+            self.objs.append(obj)
+    
+    def _load(self, nobjects=None):
+        # (TEST) use objects in healpix:
+        pix = 48656
+        self.nav = 15005
+        self.avGal = extAv(self.nav, seed=self.seed_Av)
+        self.objs = []
+
+        gals_cat = h5.File(self.galaxy_path, 'r')['galaxies']
+        gals = gals_cat[str(pix)]
+        self._load_gals(gals, pix_id=pix, nobjects=nobjects)
+        del gals
+
+    def load_sed(self, obj, **kwargs):
+        if obj.type == 'star':
+            _, wave, flux = tag_sed(
+                h5file=self.tempSED_star,
+                model_tag=obj.param['model_tag'],
+                teff=obj.param['teff'],
+                logg=obj.param['logg'],
+                feh=obj.param['feh']
+            )
+        elif obj.type == 'galaxy' or obj.type == 'quasar':
+            sed_data = getObservedSED(
+                sedCat=self.tempSed_gal[obj.sed_type],
+                redshift=obj.z,
+                av=obj.param["av"],
+                redden=obj.param["redden"]
+            )
+            wave, flux = sed_data[0], sed_data[1]
+        else:
+            raise ValueError("Object type not known")
+        speci = interpolate.interp1d(wave, flux)
+        lamb = np.arange(2000, 18001 + 0.5, 0.5)
+        y = speci(lamb)
+        # erg/s/cm2/A --> photo/s/m2/A
+        all_sed = y * lamb / (cons.h.value * cons.c.value) * 1e-13
+        sed = Table(np.array([lamb, all_sed]).T, names=('WAVELENGTH', 'FLUX'))
+        del wave
+        del flux
+        return sed
\ No newline at end of file

README.md

-# injection_pipeline
-
-
-
-## Getting started
-
-To make it easy for you to get started with GitLab, here's a list of recommended next steps.
-
-Already a pro? Just edit this README.md and make it your own. Want to make it easy? [Use the template at the bottom](#editing-this-readme)!
-
-## Add your files
-
-- [ ] [Create](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#create-a-file) or [upload](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#upload-a-file) files
-- [ ] [Add files using the command line](https://docs.gitlab.com/ee/gitlab-basics/add-file.html#add-a-file-using-the-command-line) or push an existing Git repository with the following command:
-
-```
-cd existing_repo
-git remote add origin https://csst-tb.bao.ac.cn/code/fangyuedong/injection_pipeline.git
-git branch -M main
-git push -uf origin main
-```
-
-## Integrate with your tools
-
-- [ ] [Set up project integrations](http://10.3.10.28/code/fangyuedong/injection_pipeline/-/settings/integrations)
-
-## Collaborate with your team
-
-- [ ] [Invite team members and collaborators](https://docs.gitlab.com/ee/user/project/members/)
-- [ ] [Create a new merge request](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html)
-- [ ] [Automatically close issues from merge requests](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#closing-issues-automatically)
-- [ ] [Enable merge request approvals](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/)
-- [ ] [Automatically merge when pipeline succeeds](https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html)
-
-## Test and Deploy
-
-Use the built-in continuous integration in GitLab.
-
-- [ ] [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/quick_start/index.html)
-- [ ] [Analyze your code for known vulnerabilities with Static Application Security Testing(SAST)](https://docs.gitlab.com/ee/user/application_security/sast/)
-- [ ] [Deploy to Kubernetes, Amazon EC2, or Amazon ECS using Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/requirements.html)
-- [ ] [Use pull-based deployments for improved Kubernetes management](https://docs.gitlab.com/ee/user/clusters/agent/)
-- [ ] [Set up protected environments](https://docs.gitlab.com/ee/ci/environments/protected_environments.html)
-
-***
-
-# Editing this README
-
-When you're ready to make this README your own, just edit this file and use the handy template below (or feel free to structure it however you want - this is just a starting point!). Thank you to [makeareadme.com](https://www.makeareadme.com/) for this template.
-
-## Suggestions for a good README
-Every project is different, so consider which of these sections apply to yours. The sections used in the template are suggestions for most open source projects. Also keep in mind that while a README can be too long and detailed, too long is better than too short. If you think your README is too long, consider utilizing another form of documentation rather than cutting out information.
-
-## Name
-Choose a self-explaining name for your project.
-
-## Description
-Let people know what your project can do specifically. Provide context and add a link to any reference visitors might be unfamiliar with. A list of Features or a Background subsection can also be added here. If there are alternatives to your project, this is a good place to list differentiating factors.
-
-## Badges
-On some READMEs, you may see small images that convey metadata, such as whether or not all the tests are passing for the project. You can use Shields to add some to your README. Many services also have instructions for adding a badge.
-
-## Visuals
-Depending on what you are making, it can be a good idea to include screenshots or even a video (you'll frequently see GIFs rather than actual videos). Tools like ttygif can help, but check out Asciinema for a more sophisticated method.
-
-## Installation
-Within a particular ecosystem, there may be a common way of installing things, such as using Yarn, NuGet, or Homebrew. However, consider the possibility that whoever is reading your README is a novice and would like more guidance. Listing specific steps helps remove ambiguity and gets people to using your project as quickly as possible. If it only runs in a specific context like a particular programming language version or operating system or has dependencies that have to be installed manually, also add a Requirements subsection.
-
-## Usage
-Use examples liberally, and show the expected output if you can. It's helpful to have inline the smallest example of usage that you can demonstrate, while providing links to more sophisticated examples if they are too long to reasonably include in the README.
-
-## Support
-Tell people where they can go to for help. It can be any combination of an issue tracker, a chat room, an email address, etc.
-
-## Roadmap
-If you have ideas for releases in the future, it is a good idea to list them in the README.
-
-## Contributing
-State if you are open to contributions and what your requirements are for accepting them.
-
-For people who want to make changes to your project, it's helpful to have some documentation on how to get started. Perhaps there is a script that they should run or some environment variables that they need to set. Make these steps explicit. These instructions could also be useful to your future self.
-
-You can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser.
-
-## Authors and acknowledgment
-Show your appreciation to those who have contributed to the project.
-
-## License
-For open source projects, say how it is licensed.
-
-## Project status
-If you have run out of energy or time for your project, put a note at the top of the README saying that development has slowed down or stopped completely. Someone may choose to fork your project or volunteer to step in as a maintainer or owner, allowing your project to keep going. You can also make an explicit request for maintainers.
+Pipeline for source injection
\ No newline at end of file

SingleEpochImage.py

+import numpy as np
+import copy
+from astropy import wcs
+from astropy.io import fits
+import galsim
+
+from ObservationSim.Instrument import Chip, Filter, FilterParam, FocalPlane, Telescope
+from ObservationSim.PSF import PSFGauss, PSFInterp
+
+class SingleEpochImage(object):
+    
+    def __init__(self, config, filepath):
+        self.header0, self.header_img, self.img = self.read_initial_image(filepath)
+        self._get_wcs(self.header_img)
+        self._determine_unique_area(config)
+        self.output_img_fname = config['output_img_name']
+
+        if config['n_objects'] is not None:
+            # Fixed number of objects per image
+            self.objs_per_real = config['n_objects']
+        elif config['object_density'] is not None:
+            # Fixed number density of objects
+            self.objs_per_real = round(self.u_area * config['object_density'])
+        else:
+            # Grid types: calculate nobjects later
+            self.objs_per_real = None
+
+        self.tel = Telescope()
+        # Determine which CCD
+        self.chip_ID = int(self.header0['DETECTOR'][-2:])
+        # Determine epxosure time
+        self.exp_time = float(self.header0['EXPTIME'])
+        config["obs_setting"]={}
+        config["obs_setting"]["exp_time"] = self.exp_time
+        # Construnct Chip object
+        self.chip = Chip(chipID=self.chip_ID, config=config)
+        # Load PSF model
+        if config["psf_setting"]["psf_model"] == "Gauss":
+            self.psf_model = PSFGauss(chip=self.chip)
+        elif config["psf_setting"]["psf_model"] == "Interp":
+            self.psf_model = PSFInterp(chip=self.chip, PSF_data_file=config["psf_setting"]["psf_dir"])
+
+        filter_id, filter_type = self.chip.getChipFilter()
+        filter_param = FilterParam()
+        self.filt = Filter(filter_id=filter_id,
+                        filter_type=filter_type,
+                        filter_param=filter_param)
+        self.focal_plane = FocalPlane()
+        
+        self.setup_image_for_injection()
+
+    def setup_image_for_injection(self):
+        ra_cen = self.wcs.wcs.crval[0]
+        dec_cen = self.wcs.wcs.crval[1]
+        self.wcs_fp = self.focal_plane.getTanWCS(ra_cen, dec_cen, self.pos_ang*galsim.degrees, self.pixel_scale)
+        # self.inj_img = galsim.ImageF(self.chip.npix_x, self.chip.npix_y)
+        self.chip.img = galsim.Image(self.img, copy=True)
+        self.chip.img.setOrigin(self.chip.bound.xmin, self.chip.bound.ymin)
+        self.chip.img.wcs = self.wcs_fp
+        print(self.chip.img.array)
+
+    def read_initial_image(self, filepath):
+        data = fits.open(filepath)
+        header0 = data[0].header
+        header1 = data[1].header
+        image = fits.getdata(filepath)
+
+        # (TEMP)
+        image = np.float64(image)
+        image *= 1.1
+        image -= 500.
+
+        temp_img = galsim.Image(image, copy=True)
+
+        temp_img.array[temp_img.array > 65535] = 65535
+        temp_img.replaceNegative(replace_value=0)
+        temp_img.quantize()
+        temp_img = galsim.Image(temp_img.array, dtype=np.uint16)
+        # self.chip.img = galsim.Image(self.chip.img.array, dtype=np.int32)
+        hdu1 = fits.PrimaryHDU(header=header0)
+        hdu2 = fits.ImageHDU(temp_img.array, header=header1)
+        hdu1 = fits.HDUList([hdu1, hdu2])
+        fname = "nullwt_image_for_injection.fits"
+        hdu1.writeto(fname, output_verify='ignore', overwrite=True)
+        return header0, header1, image
+
+    def _get_wcs(self, header):
+        crpix1 = float(header['CRPIX1'])
+        crpix2 = float(header['CRPIX2'])
+        
+        crval1 = float(header['CRVAL1'])
+        crval2 = float(header['CRVAL2'])
+
+        ctype1 = str(header['CTYPE1'])
+        ctype2 = str(header['CTYPE2'])
+
+        cd1_1 = float(header['CD1_1'])
+        cd1_2 = float(header['CD1_2'])
+        cd2_1 = float(header['CD2_1'])
+        cd2_2 = float(header['CD2_2'])
+        self.pos_ang = float(header['POS_ANG'])
+
+        # Create WCS object
+        self.wcs = wcs.WCS()
+        self.wcs.wcs.crpix = [crpix1, crpix2]
+        self.wcs.wcs.crval = [crval1, crval2]
+        self.wcs.wcs.ctype = [ctype1, ctype2]
+        self.wcs.wcs.cd = [[cd1_1, cd1_2], [cd2_1, cd2_2]]
+
+        self.pixel_scale = 0.074
+        self.Npix_x = int(header['NAXIS1'])
+        self.Npix_y = int(header['NAXIS2'])
+
+    def _determine_unique_area(self, config):
+        coners = np.array([(1, 1), (1, self.Npix_y), (self.Npix_x, 1), (self.Npix_x, self.Npix_y)])
+        coners = self.wcs.wcs_pix2world(coners, 1)
+        ra_coners = coners[:, 0]
+        dec_coners = coners[:, 1]
+        self.ramin, self.ramax = min(ra_coners), max(ra_coners)
+        self.decmin, self.decmax = min(dec_coners), max(dec_coners)
+
+        if self.ramax - self.ramin > 1.:
+            self.ra_boundary_cross = True
+        else:
+            self.ra_boundary_cross = False
+        
+        d1, d2 = np.deg2rad([self.decmin, self.decmax])
+        r1, r2 = self.ramin, self.ramax
+
+        if self.ra_boundary_cross:
+            r2 = r2 + 360.
+        
+        # In deg^2
+        a = (180. / np.pi) * (r2 - r1) * (np.sin(d2) - np.sin(d1))
+        # Save in arcmin^2
+        self.u_area = 3600. * a
+
+    def inject_objects(self, pos, cat):
+        nobj = len(pos)
+        # Make sure we have enough objects to inject
+        assert nobj <= len(cat.objs)
+        
+        for i in range(nobj):
+            obj = cat.objs[i]
+            try:
+                sed_data = cat.load_sed(obj)
+                norm_filt = cat.load_norm_filt(obj)
+                obj.sed, obj.param["mag_%s"%self.filt.filter_type], obj.param["flux_%s"%self.filt.filter_type] = cat.convert_sed(
+                        mag=obj.param["mag_use_normal"],
+                        sed=sed_data,
+                        target_filt=self.filt, 
+                        norm_filt=norm_filt)
+            except Exception as e:
+                print(e)
+                continue
+
+            # Update object position to a point on grid
+            obj.param['ra'], obj.param['dec'] = pos[i][0], pos[i][1]
+
+            pos_img, offset, local_wcs = obj.getPosImg_Offset_WCS(img=self.chip.img)
+            print(pos_img.x, pos_img.y)
+            try:
+                isUpdated, pos_shear = obj.drawObj_multiband(
+                    tel=self.tel,
+                    pos_img=pos_img, 
+                    psf_model=self.psf_model, 
+                    bandpass_list=self.filt.bandpass_sub_list,
+                    filt=self.filt, 
+                    chip=self.chip, 
+                    g1=obj.g1, 
+                    g2=obj.g2, 
+                    exptime=self.exp_time)
+                if isUpdated:
+                    # TODO: add up stats
+                    # print("updating output catalog...")
+                    print('Updated')
+                    pass
+                else:
+                    # print("object omitted", flush=True)
+                    continue
+            except Exception as e:
+                print(e)
+                pass
+            # Unload SED:
+            obj.unload_SED()
+            del obj
+
+    def save_injected_img(self):
+        self.chip.img.array[self.chip.img.array > 65535] = 65535
+        self.chip.img.replaceNegative(replace_value=0)
+        self.chip.img.quantize()
+        self.chip.img = galsim.Image(self.chip.img.array, dtype=np.uint16)
+        # self.chip.img = galsim.Image(self.chip.img.array, dtype=np.int32)
+        hdu1 = fits.PrimaryHDU(header=self.header0)
+        hdu2 = fits.ImageHDU(self.chip.img.array, header=self.header_img)
+        hdu1 = fits.HDUList([hdu1, hdu2])
+        # fname = 'test_inject.fits'
+        # fname = '20220621_test_injection.fits'
+        fname = self.output_img_fname
+        hdu1.writeto(fname, output_verify='ignore', overwrite=True)
+
+            

config.py

+import argparse
+
+def parse_args():
+    '''
+    Parse command line arguments. Many of the following
+    can be set in the .yaml config file as well.
+    '''
+    parser = argparse.ArgumentParser()
+    parser.add_argument('config_file', help='.yaml config file for injection settings.')
+    parser.add_argument('-c', '--config_dir', help='Directory that houses the .yaml config file.')
+    # parser.add_argument('-d', '--data_dir', help='Directory that houses the input data.')
+    # parser.add_argument('-w', '--work_dir', help='The path for output.')
+    return parser.parse_args()
\ No newline at end of file

config_injection.yaml

+---
+###############################################
+#
+#  Configuration file for CSST object injection
+#      Last modified: 2022/06/19
+#
+###############################################
+n_objects: 20
+rotate_objs: False
+pos_sampling:
+  # type: "HexGrid"
+  type: "RectGrid"
+  grid_spacing: 74 # arcsec (~1000 pixels)
+
+# output_img_name: "test_HexGrid_20220628.fits"
+output_img_name: "test_RectGrid_20220628.fits"
+###############################################
+# PSF setting
+###############################################
+psf_setting:
+  
+  # Which PSF model to use:
+  # "Gauss": simple gaussian profile
+  # "Interp": Interpolated PSF from sampled ray-tracing data
+  psf_model: "Interp"
+
+  # PSF size [arcseconds]
+  # radius of 80% energy encircled
+  # NOTE: only valid for "Gauss" PSF
+  psf_rcont: 0.15
+
+  # path to PSF data
+  # NOTE: only valid for "Interp" PSF
+  psf_dir: "/data/simudata/CSSOSDataProductsSims/data/csstPSFdata/psfCube"
+
+###############################################
+# Input path setting
+# (NOTE) Used NGP Catalog for testing
+###############################################
+# Default path settings for NGP footprint simulation
+data_dir: "/data/simudata/CSSOSDataProductsSims/data/"
+
+input_path:
+  cat_dir: "OnOrbitCalibration/CTargets20211231"
+  star_cat: "CT-NGP_r1.8_G28.hdf5"
+  galaxy_cat: "galaxyCats_r_3.0_healpix_shift_192.859500_27.128300.hdf5"
+
+SED_templates_path:
+  star_SED: "Catalog_20210126/SpecLib.hdf5"
+  galaxy_SED: "Templates/Galaxy/"
+
+
+###############################################
+# Instrumental effects setting
+# (NOTE) Here only used to construct
+# ObservationSim.Instrument.Chip object
+# (TODO) Should readout from header
+###############################################
+ins_effects:
+  # switches
+  bright_fatter:  ON # Whether to simulate Brighter-Fatter (also diffusion) effect
+
+  # values
+  dark_exptime:   300   # Exposure time for dark current frames [seconds]
+  flat_exptime:   150   # Exposure time for flat-fielding frames [seconds]
+  readout_time:   40    # The read-out time for each channel [seconds]
+  df_strength:    2.3   # Sillicon sensor diffusion strength
+  bias_level:     500   # bias level [e-/pixel]
+  gain:           1.1   # Gain
+  full_well:      90000 # Full well depth [e-]
+
+###############################################
+# Random seeds
+###############################################
+random_seeds:
+  seed_Av:              121212    # Seed for generating random intrinsic extinction
\ No newline at end of file

injection_pipeline.py

+from astropy.io import fits
+from astropy import wcs
+import galsim
+import os
+import yaml
+
+from SingleEpochImage import SingleEpochImage
+from InjectionCatalog import InjectionCatalog
+from InputCatalogs import SimCat
+from Grid import RectGrid
+from config import parse_args
+
+# img_file = 'CSST_MSC_MS_SCI_20240617065639_20240617065909_100000000_24_L0_1.fits'
+img_file = 'CSST_MSC_MS_SCI_20240617065639_20240617065909_100000000_23_L0_1.fits' # null-weight test image
+
+args = parse_args()
+if args.config_dir is None:
+    args.config_dir = ''
+args.config_dir = os.path.abspath(args.config_dir)
+args.config_file = os.path.join(args.config_dir, args.config_file)
+print(args.config_file)
+with open(args.config_file, "r") as stream:
+    try:
+        config = yaml.safe_load(stream)
+        for key, value in config.items():
+            print (key + " : " + str(value))
+    except yaml.YAMLError as exc:
+        print(exc)
+
+image = SingleEpochImage(config=config, filepath=img_file)
+# print(image.ramin, image.ramax, image.u_area, image.ra_boundary_cross)
+inject_cat = InjectionCatalog(image=image)
+inject_cat.generate_positions(config=config)
+input_cat = SimCat(config=config, chip=image.chip, nobjects=inject_cat.nobjects)
+print(inject_cat.pos, inject_cat.nobjects)
+image.inject_objects(pos=inject_cat.pos, cat=input_cat)
+image.save_injected_img()
\ No newline at end of file

mathutil.py

+import numpy as np
+
+#-------------------------------------------------------------------------------
+# Unit conversions and misc functions
+
+def deg2arcmin(val):
+    return 60.0 * val
+
+def arcmin2deg(val):
+    return val / 60.0
+
+def sample_uniform_ra(r1, r2, N=None, boundary_cross=False):
+    '''
+    Sample N random RA values from r1 to r2, where r1<r2. If boundary_cross
+    is set to True, will adjust inputs to allow sampling over 360/0 boundary.
+    '''
+
+    # Adjust r1, r2 if they cross the 360/0 ra boundary
+    if boundary_cross is True:
+        # NOTE: ramin/ramax are saved the same way as the geometry file catalog,
+        # which is to say ramin/ramax are always 'left'/'right' boundary, even if
+        # ramax < ramin across boundary.
+        # This means that r1, while 'min', is larger and needs to be adjusted down.
+        r1_shift = r1 - 360.0
+        shifted_dist = sample_uniform(r1_shift, r2, N)
+        shifted_dist[shifted_dist < 0.0] += 360.0
+        return shifted_dist
+
+    else:
+        # NOTE: Equivalent to below DEC procedure for ra = P(r2-r1)+r1 for P in (0,1)
+        return sample_uniform(r1, r2, N)
+
+def sample_uniform_dec(d1, d2, N=None, unit='deg'):
+    '''
+    Sample N random DEC values from d1 to d2, accounting for curvature of sky.
+    '''
+
+    if N is None: N=1
+
+    # Convert dec to radians if not already
+    if unit == 'deg' or unit == 'degree':
+        d1, d2 = np.deg2rad(d1), np.deg2rad(d2)
+
+    # Uniform sampling from 0 to 1
+    P = np.random.rand(N)
+
+    # Can't use `sample_uniform()` as dec needs angular weighting
+    delta = np.arcsin(P * (np.sin(d2) - np.sin(d1)) +np.sin(d1))
+
+    # Return correct unit
+    if unit is 'deg' or unit is 'degree':
+        return np.rad2deg(delta)
+    elif unit is 'rad' or unit is 'radian':
+        return delta
+    else:
+        raise TypeError('Only deg/degree or rad/radian are allowed as inputs!')
+
+def sample_uniform_indx(n1, n2, N=None):
+    'Samples N random indices from n1 to n2'
+
+    return np.random.randint(n1, high=n2, size=N)
+
+def sample_uniform(v1, v2, N=None):
+    'Samples N random values from v1 to v2'
+
+    if N is None: N=1
+
+    return np.random.uniform(low=v1, high=v2, size=N)

run.sh

+#!/bin/bash
+
+python /public/home/fangyuedong/test/injection_pipeline/injection_pipeline.py config_injection.yaml -c /public/home/fangyuedong/test/injection_pipeline/
\ No newline at end of file