From 3499eac20115bcd91cfd7b1cfb291033b0d26d6d Mon Sep 17 00:00:00 2001 From: fangyuedong Date: Sun, 14 Apr 2024 05:18:56 +0800 Subject: [PATCH] * unified the naming of .cat and .log files. * Changed the name of output folder of pointing to corresponding pointing.obs_id. * fixed the order .cat hdr updating and the creation of .cat file. * Changed the naming of shutter and flat reference image: chip IDs now all in 2 digits. * Updated the __init__ method of ChipOutput * Removed Config.py from ObservationSim.Config * Cleaned codes in ObservationSim.py, ChipOutput --- Catalog/C6_50sqdeg.py | 2 +- Catalog/C6_Catalog.py | 2 +- Catalog/C6_fits_Catalog.py | 2 +- Catalog/Calib_Catalog.py | 2 +- Catalog/FGS_Catalog.py | 2 +- Catalog/NGPCatalog.py | 2 +- Catalog/fd_test_C6.py | 2 +- Catalog/testCat_fits.py | 2 +- Catalog/testCat_galaxy.py | 2 +- Catalog/testCat_star.py | 2 +- Catalog/wcs_test_C6.py | 2 +- ObservationSim/Config/ChipOutput.py | 58 +++--- ObservationSim/Config/Config.py | 93 --------- ObservationSim/Config/Pointing.py | 5 +- ObservationSim/Config/__init__.py | 1 - ObservationSim/ObservationSim.py | 178 ++---------------- ObservationSim/sim_steps/add_objects.py | 8 +- ObservationSim/sim_steps/add_pattern_noise.py | 2 +- .../sim_steps/add_sky_background.py | 2 +- config/obs_config_SCI_WIDE_phot.yaml | 5 +- .../test_PSFInterpModule_coverage.py | 4 - tests/PSFInterpTest/test_loadPSFSet.py | 4 - tests/TestSpecDisperse.py | 4 +- 23 files changed, 66 insertions(+), 320 deletions(-) delete mode 100755 ObservationSim/Config/Config.py diff --git a/Catalog/C6_50sqdeg.py b/Catalog/C6_50sqdeg.py index 8335bfa..4ce9f3c 100644 --- a/Catalog/C6_50sqdeg.py +++ b/Catalog/C6_50sqdeg.py @@ -122,7 +122,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/C6_Catalog.py b/Catalog/C6_Catalog.py index 7d114e5..4814945 100644 --- a/Catalog/C6_Catalog.py +++ b/Catalog/C6_Catalog.py @@ -99,7 +99,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/C6_fits_Catalog.py b/Catalog/C6_fits_Catalog.py index 0f03dc4..d8eea0c 100644 --- a/Catalog/C6_fits_Catalog.py +++ b/Catalog/C6_fits_Catalog.py @@ -113,7 +113,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/Calib_Catalog.py b/Catalog/Calib_Catalog.py index 3987ef5..308a9d5 100644 --- a/Catalog/Calib_Catalog.py +++ b/Catalog/Calib_Catalog.py @@ -106,7 +106,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/FGS_Catalog.py b/Catalog/FGS_Catalog.py index 3b0c230..bdab18c 100644 --- a/Catalog/FGS_Catalog.py +++ b/Catalog/FGS_Catalog.py @@ -100,7 +100,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/NGPCatalog.py b/Catalog/NGPCatalog.py index f37b73c..cf6ae97 100644 --- a/Catalog/NGPCatalog.py +++ b/Catalog/NGPCatalog.py @@ -65,7 +65,7 @@ class Catalog(CatalogBase): def _add_output_columns_header(self): self.add_hdr = " model_tag teff logg feh" self.add_fmt = " %10s %8.4f %8.4f %8.4f" - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/fd_test_C6.py b/Catalog/fd_test_C6.py index 0b7909a..cbb41e8 100644 --- a/Catalog/fd_test_C6.py +++ b/Catalog/fd_test_C6.py @@ -99,7 +99,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/testCat_fits.py b/Catalog/testCat_fits.py index 2c73c50..b148431 100644 --- a/Catalog/testCat_fits.py +++ b/Catalog/testCat_fits.py @@ -77,7 +77,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/testCat_galaxy.py b/Catalog/testCat_galaxy.py index 8ecd698..fd9a5a4 100644 --- a/Catalog/testCat_galaxy.py +++ b/Catalog/testCat_galaxy.py @@ -97,7 +97,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/testCat_star.py b/Catalog/testCat_star.py index 50207da..458e41c 100644 --- a/Catalog/testCat_star.py +++ b/Catalog/testCat_star.py @@ -124,7 +124,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/wcs_test_C6.py b/Catalog/wcs_test_C6.py index 896ab4a..5623ac2 100644 --- a/Catalog/wcs_test_C6.py +++ b/Catalog/wcs_test_C6.py @@ -99,7 +99,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/ObservationSim/Config/ChipOutput.py b/ObservationSim/Config/ChipOutput.py index 4ad0762..5d11380 100755 --- a/ObservationSim/Config/ChipOutput.py +++ b/ObservationSim/Config/ChipOutput.py @@ -1,35 +1,40 @@ import os import logging import ObservationSim.Config._util as _util +from ObservationSim.Config.Header import generatePrimaryHeader class ChipOutput(object): - def __init__(self, config, focal_plane, chip, filt, imgKey0="", imgKey1="", imgKey2="", exptime=150., mjdTime="", ra_cen=None, dec_cen=None, pointing_type='SCI', pointing_ID='0',pointing_type_code='101', subdir="./", prefix=""): - self.focal_plane = focal_plane + def __init__(self, config, chip, filt, pointing): + self.config = config self.chip = chip self.filt = filt - self.imgKey0 = imgKey0 - self.imgKey1 = imgKey1 - self.imgKey2 = imgKey2 - self.exptime = exptime - self.mjdTime = mjdTime - self.pointing_type = pointing_type - if (ra_cen is not None) and (dec_cen is not None): - self.ra_cen = ra_cen - self.dec_cen = dec_cen - else: - self.ra_cen = config["obs_setting"]["ra_center"] - self.dec_cen = config["obs_setting"]["dec_center"] + self.pointing_type = pointing.pointing_type + self.chip_label = str(chip.chipID).rjust(2, '0') - self.chipLabel = focal_plane.getChipLabel(chip.chipID) - obs_id = _util.get_obs_id(img_type=pointing_type, project_cycle=config["project_cycle"], run_counter=config["run_counter"], pointing_id=pointing_ID, pointing_type_code = pointing_type_code) - # self.cat_name = "MSC_1%s_chip_%s_filt_%s"%(str(pointing_ID).rjust(8, '0'), self.chipLabel, filt.filter_type) + ".cat" - self.cat_name = "MSC_%s_chip_%s_filt_%s"%(obs_id, self.chipLabel, filt.filter_type) + ".cat" + # Get primary header based on chip and pointing + self.h_prim = generatePrimaryHeader( + xlen=chip.npix_x, + ylen=chip.npix_y, + pointing_id = pointing.obs_id, + pointing_type_code = pointing.pointing_type_code, + ra=pointing.ra, + dec=pointing.dec, + pixel_scale=chip.pix_scale, + time_pt = pointing.timestamp, + exptime=pointing.exp_time, + im_type=pointing.pointing_type, + sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z], + sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz], + project_cycle=self.config["project_cycle"], + run_counter=self.config["run_counter"], + chip_name=self.chip_label) - self.subdir = subdir + obs_id = _util.get_obs_id(img_type=self.pointing_type, project_cycle=config["project_cycle"], run_counter=config["run_counter"], pointing_id=pointing.obs_id, pointing_type_code = pointing.pointing_type_code) + + self.subdir = pointing.output_dir + self.cat_name = self.h_prim['FILENAME'] + '.cat' + logger_filename = self.h_prim['FILENAME'] + '.log' - # Setup logger for each chip - # logger_filename = "MSC_1%s_chip_%s_filt_%s"%(str(pointing_ID).rjust(8, '0'), self.chipLabel, filt.filter_type) + ".log" - logger_filename = "MSC_%s_chip_%s_filt_%s"%(obs_id, self.chipLabel, filt.filter_type) + ".log" self.logger = logging.getLogger() fh = logging.FileHandler(os.path.join(self.subdir, logger_filename), mode='w+', encoding='utf-8') fh.setLevel(logging.DEBUG) @@ -46,7 +51,7 @@ class ChipOutput(object): self.hdr = hdr1 + hdr2 self.fmt = fmt1 + fmt2 - self.logger.info("pointing_type = %s\n"%(pointing_type)) + self.logger.info("pointing_type = %s\n"%(self.pointing_type)) def Log_info(self, message): print(message) @@ -56,7 +61,7 @@ class ChipOutput(object): print(message) self.logger.error(message) - def update_ouptut_header(self, additional_column_names=""): + def update_output_header(self, additional_column_names=""): self.hdr += additional_column_names def create_output_file(self): @@ -68,14 +73,11 @@ class ChipOutput(object): self.cat.write(self.hdr) def cat_add_obj(self, obj, pos_img, pos_shear): - # self.logger.info('-------------debug-----------------') - # self.logger.info('from global',ximg, yimg) ximg = obj.real_pos.x + 1.0 yimg = obj.real_pos.y + 1.0 - # self.logger.info('from loacl',ximg, yimg) line = self.fmt%( - obj.id, int(self.chipLabel), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.ra_orig, obj.dec_orig, obj.z, obj.getMagFilter(self.filt), obj.type, + obj.id, int(self.chip_label), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.ra_orig, obj.dec_orig, obj.z, obj.getMagFilter(self.filt), obj.type, obj.pmra, obj.pmdec, obj.rv, obj.parallax) line += obj.additional_output_str if not line.endswith("\n"): diff --git a/ObservationSim/Config/Config.py b/ObservationSim/Config/Config.py deleted file mode 100755 index 8512026..0000000 --- a/ObservationSim/Config/Config.py +++ /dev/null @@ -1,93 +0,0 @@ -import galsim -import os -from astropy.time import Time as asTime - -def config_dir(config, work_dir=None, data_dir=None): - path_dict = {} - # Working directory - if work_dir == None: - dirname, _ = os.path.split(os.path.abspath(__file__)) - path_dict["work_dir"] = "/".join(dirname.split("/")[:-1]) + "/" - else: - path_dict["work_dir"] = work_dir - - # Data directory - # if data_dir == None: - # # Assume all input datasets are in the work directory - # path_dict["data_dir"] =os.path.join(path_dict["work_dir"], "data/") - # else: - # path_dict["data_dir"] = data_dir - - # PSF data directory - # if config["psf_setting"]["psf_model"] == "Interp": - # path_dict["psf_pho_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_pho_dir"]) - # path_dict["psf_sls_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_sls_dir"]) - if config["psf_setting"]["psf_model"] == "Interp": - path_dict["psf_pho_dir"] = config["psf_setting"]["psf_pho_dir"] - path_dict["psf_sls_dir"] = config["psf_setting"]["psf_sls_dir"] - - return path_dict - -def read_config(config_filename): - """Read in a configuration file and return the corresponding dict(s). - - Parameters: - config_filename: The name of the configuration file to read. - Returns: - (list) of config dicts - """ - config = {} - config_file = open(config_filename).readlines() - nlines = len(config_file) - for i in range(nlines): - row = config_file[i].split() - if len(row) <= 1: continue # blank row - if not "#" in row: - if len(row) == 2: - key, val = row[0:2] - config.update({key:val}) - else: - print("!! Something is wrong with parameter '%s'."%row[0]) - return - elif row.index("#") == 2: - key, val = row[0:2] - config.update({key:val}) - elif row.index("#") == 0: - continue # annotation - else: - print("!! Something is wrong with parameter '%s'."%row[0]) - return - config = parse_config(config) - return config - -def parse_config(config): - """Parse the config values to the right type - - Parameters: - config: raw config dict - Returns: - Parsed config dict - """ - config["ra_center"] = float(config["ra_center"]) - config["dec_center"] = float(config["dec_center"]) - config["psf_rcont"] = config["psf_rcont"].split(",") - config["psfRa"] = float(config["psf_rcont"][0]) - config["psfCont"] = float(config["psf_rcont"][1]) - config["image_rot"] = float(config["image_rot"])*galsim.degrees - config["sigma_spin"] = float(config["sigma_spin"]) - config["reduced_g1"] = float(config["reduced_g1"]) - config["reduced_g2"] = float(config["reduced_g2"]) - config["rotateEll"] = float(config["rotateEll"]) - config["reEll"] = int(config["rotateEll"]/45.0) - if config["reEll"]==0: config["reIndex"] = "P" - if config["reEll"]==1: config["reIndex"] = "X" - if config["reEll"]==2: config["reIndex"] = "N" - if config["reEll"]==3: config["reIndex"] = "Y" - config["seed_flat"] = int(config["seed_flat"]) - config["seed_prnu"] = int(config["seed_prnu"]) - config["seed_star"] = int(config["seed_star"]) - config["seed_gal"] = int(config["seed_gal"]) - config["seed_Av"] = int(config["seed_Av"]) - config["bias_level"] = int(config["bias_level"]) - config["df_strength"] = float(config["df_strength"]) - return config diff --git a/ObservationSim/Config/Pointing.py b/ObservationSim/Config/Pointing.py index 9eb0f1a..6d95b4d 100644 --- a/ObservationSim/Config/Pointing.py +++ b/ObservationSim/Config/Pointing.py @@ -27,6 +27,7 @@ class Pointing(object): self.obs_config_file = obs_config_file self.t_shutter_open = t_shutter_open self.t_shutter_close = t_shutter_close + self.output_dir = "." if self.obs_config_file is not None: with open(self.obs_config_file, "r") as stream: try: @@ -107,7 +108,7 @@ class Pointing(object): else: self.timestamp = t - def make_output_pointing_dir(self, overall_config, pointing_ID=0, copy_obs_config=False): + def make_output_pointing_dir(self, overall_config, copy_obs_config=False): run_dir = os.path.join(overall_config["work_dir"], overall_config["run_name"]) if not os.path.exists(run_dir): try: @@ -131,7 +132,7 @@ class Pointing(object): obs_config_output_path = os.path.join(self.output_dir, os.path.basename(self.obs_config_file)) if not os.path.exists(obs_config_output_path): try: - shutil.copy(args.obs_config_file, self.output_dir) + shutil.copy(self.obs_config_file, self.output_dir) except OSError: pass diff --git a/ObservationSim/Config/__init__.py b/ObservationSim/Config/__init__.py index 96ca93e..bf03e61 100644 --- a/ObservationSim/Config/__init__.py +++ b/ObservationSim/Config/__init__.py @@ -1,3 +1,2 @@ -from .Config import * from .ChipOutput import ChipOutput from .Pointing import Pointing \ No newline at end of file diff --git a/ObservationSim/ObservationSim.py b/ObservationSim/ObservationSim.py index 3421713..3d8030d 100755 --- a/ObservationSim/ObservationSim.py +++ b/ObservationSim/ObservationSim.py @@ -8,17 +8,15 @@ from datetime import datetime import traceback -from ObservationSim.Config import config_dir, ChipOutput +from ObservationSim.Config import ChipOutput from ObservationSim.Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip from ObservationSim.Instrument.Chip import Effects from ObservationSim.Instrument.Chip import ChipUtils as chip_utils -from ObservationSim._util import make_output_pointing_dir from ObservationSim.Astrometry.Astrometry_util import on_orbit_obs_position from ObservationSim.sim_steps import SimSteps, SIM_STEP_TYPES class Observation(object): def __init__(self, config, Catalog, work_dir=None, data_dir=None): - self.path_dict = config_dir(config=config, work_dir=work_dir, data_dir=data_dir) self.config = config self.tel = Telescope() self.filter_param = FilterParam() @@ -92,9 +90,6 @@ class Observation(object): # Prepare necessary chip properties for simulation chip = self.prepare_chip_for_exposure(chip, ra_cen, dec_cen, pointing) - # Load catalogues - # self.cat = self.Catalog(config=self.config, chip=chip, pointing=pointing, cat_dir=cat_dir, sed_dir=sed_dir, chip_output=chip_output, filt=filt) - # Initialize SimSteps sim_steps = SimSteps(overall_config=self.config, chip_output=chip_output, all_filters=self.all_filters) @@ -124,138 +119,6 @@ class Observation(object): chip_output.Log_info("check running:1: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) )) del chip.img - # def run_one_chip_calibration(self, chip, filt, pointing, chip_output, skyback_level = 20000, sky_level_filt = 'g', wcs_fp=None, psf_model=None, cat_dir=None, sed_dir=None): - - - - - # # # Get WCS for the focal plane - # # if wcs_fp == None: - # # wcs_fp = self.focal_plane.getTanWCS(ra_cen, dec_cen, pointing.img_pa, chip.pix_scale) - - # # Create chip Image - # chip.img = galsim.ImageF(chip.npix_x, chip.npix_y) - # chip.img.setOrigin(chip.bound.xmin, chip.bound.ymin) - # # chip.img.wcs = wcs_fp - # pf_map = np.zeros_like(chip.img.array) - # if self.config["obs_setting"]["LED_TYPE"] is not None: - # if len(self.config["obs_setting"]["LED_TYPE"]) != 0: - # print("LED OPEN--------") - - # led_obj = FlatLED(chip, filt) - - # led_flat = led_obj.drawObj_LEDFlat(led_type_list=self.config["obs_setting"]["LED_TYPE"], exp_t_list=self.config["obs_setting"]["LED_TIME"]) - # pf_map = led_flat - # # whether to output zero, dark, flat calibration images. - # expTime = self.config["obs_setting"]["exp_time"] - # norm_scaler = skyback_level/expTime / self.filter_param.param[sky_level_filt][5] - - # if skyback_level == 0: - # self.config["ins_effects"]["shutter_effect"] = False - - # if chip.survey_type == "photometric": - # sky_map = np.ones_like(chip.img.array) * norm_scaler * self.filter_param.param[chip.filter_type][5] / self.tel.pupil_area - # elif chip.survey_type == "spectroscopic": - # flat_normal = np.ones_like(chip.img.array) - # if self.config["ins_effects"]["flat_fielding"] == True: - # chip_output.Log_info("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding" % chip.chipID) - # msg = str(chip.img.bounds) - # chip_output.Log_info(msg) - # flat_img = Effects.MakeFlatSmooth( - # chip.img.bounds, - # int(self.config["random_seeds"]["seed_flat"])) - # flat_normal = flat_normal * flat_img.array / np.mean(flat_img.array) - # if self.config["ins_effects"]["shutter_effect"] == True: - # chip_output.Log_info("SLS flat preprocess,CHIP %d : Apply shutter effect" % chip.chipID) - # shuttimg = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735, - # dt=1E-3) # shutter effect normalized image for this chip - # flat_normal = flat_normal * shuttimg - # flat_normal = np.array(flat_normal, dtype='float32') - # sky_map = calculateSkyMap_split_g( - # skyMap=flat_normal, - # blueLimit=filt.blue_limit, - # redLimit=filt.red_limit, - # conf=chip.sls_conf, - # pixelSize=chip.pix_scale, - # isAlongY=0, - # flat_cube=chip.flat_cube) - # sky_map = sky_map * norm_scaler - - # chip.img = chip.addEffects( - # config=self.config, - # img=chip.img, - # chip_output=chip_output, - # filt=filt, - # ra_cen=pointing.ra, - # dec_cen=pointing.dec, - # img_rot=pointing.img_pa, - # exptime=self.config["obs_setting"]["exp_time"], - # pointing_ID=pointing.id, - # timestamp_obs=pointing.timestamp, - # pointing_type=pointing.pointing_type, - # sky_map=sky_map, tel=self.tel, - # post_flash_map=pf_map, - # logger=chip_output.logger) - - # datetime_obs = datetime.utcfromtimestamp(pointing.timestamp) - # date_obs = datetime_obs.strftime("%y%m%d") - # time_obs = datetime_obs.strftime("%H%M%S") - # h_prim = generatePrimaryHeader( - # xlen=chip.npix_x, - # ylen=chip.npix_y, - # pointNum=str(pointing.id), - # ra=pointing.ra, - # dec=pointing.dec, - # pixel_scale=chip.pix_scale, - # date=date_obs, - # time_obs=time_obs, - # exptime=self.config["obs_setting"]["exp_time"], - # im_type='DARKPF', - # sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z], - # sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz], - # chip_name=str(chip.chipID).rjust(2, '0')) - # h_ext = generateExtensionHeader( - # chip=chip, - # xlen=chip.npix_x, - # ylen=chip.npix_y, - # ra=pointing.ra, - # dec=pointing.dec, - # pa=pointing.img_pa.deg, - # gain=chip.gain, - # readout=chip.read_noise, - # dark=chip.dark_noise, - # saturation=90000, - # pixel_scale=chip.pix_scale, - # pixel_size=chip.pix_size, - # xcen=chip.x_cen, - # ycen=chip.y_cen, - # extName='SCI', - # timestamp=pointing.timestamp, - # exptime=self.config["obs_setting"]["exp_time"], - # readoutTime=chip.readout_time) - - # chip.img = galsim.Image(chip.img.array, dtype=np.uint16) - # hdu1 = fits.PrimaryHDU(header=h_prim) - # hdu1.add_checksum() - # hdu1.header.comments['CHECKSUM'] = 'HDU checksum' - # hdu1.header.comments['DATASUM'] = 'data unit checksum' - # hdu2 = fits.ImageHDU(chip.img.array, header=h_ext) - # hdu2.add_checksum() - # hdu2.header.comments['XTENSION'] = 'extension type' - # hdu2.header.comments['CHECKSUM'] = 'HDU checksum' - # hdu2.header.comments['DATASUM'] = 'data unit checksum' - # hdu1 = fits.HDUList([hdu1, hdu2]) - # fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits') - # hdu1.writeto(fname, output_verify='ignore', overwrite=True) - - # # chip_output.Log_info("# objects that are too bright %d out of %d" % (bright_obj, self.nobj)) - # # chip_output.Log_info("# objects that are too dim %d out of %d" % (dim_obj, self.nobj)) - # # chip_output.Log_info("# objects that are missed %d out of %d" % (missed_obj, self.nobj)) - # del chip.img - - # chip_output.Log_info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB" % ( - # pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024))) - def runExposure_MPI_PointingList(self, pointing_list, chips=None, use_mpi=False): if use_mpi: comm = MPI.COMM_WORLD @@ -266,9 +129,10 @@ class Observation(object): for ipoint in range(len(pointing_list)): # Construct chips & filters: pointing = pointing_list[ipoint] - pointing_ID = pointing.id + # pointing_ID = pointing.id + pointing_ID = pointing.obs_id - sub_img_dir, prefix = make_output_pointing_dir(path_dict=self.path_dict, config=self.config, pointing_ID=pointing_ID) + pointing.make_output_pointing_dir(overall_config=self.config, copy_obs_config=True) self.focal_plane = FocalPlane(chip_list=pointing.obs_param["run_chips"]) # Make Chip & Filter lists @@ -314,39 +178,19 @@ class Observation(object): chip = run_chips[ichip] filt = run_filts[ichip] - # chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid)) + chip_output = ChipOutput( - config=self.config, - focal_plane=self.focal_plane, - chip=chip, - filt=filt, - exptime=pointing.exp_time, - pointing_type=pointing.pointing_type, - pointing_ID=pointing.obs_id, - pointing_type_code = pointing.pointing_type_code, - subdir=sub_img_dir, - prefix=prefix, - ra_cen=pointing.ra, - dec_cen=pointing.dec) - chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid)) + config = self.config, + chip = chip, + filt = filt, + pointing = pointing + ) + chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(int(pointing_ID), chip.chipID, pid)) self.run_one_chip( chip=chip, filt=filt, chip_output=chip_output, pointing=pointing) - # if self.config["obs_setting"]["survey_type"] == "CALIBRATION": - # self.run_one_chip_calibration(chip=chip, - # filt=filt, - # chip_output=chip_output, - # pointing=pointing, - # skyback_level = self.config["obs_setting"]["FLAT_LEVEL"], - # sky_level_filt = self.config["obs_setting"]["FLAT_LEVEL_FIL"]) - # else: - # self.run_one_chip( - # chip=chip, - # filt=filt, - # chip_output=chip_output, - # pointing=pointing) chip_output.Log_info("finished running chip#%d..."%(chip.chipID)) for handler in chip_output.logger.handlers[:]: chip_output.logger.removeHandler(handler) diff --git a/ObservationSim/sim_steps/add_objects.py b/ObservationSim/sim_steps/add_objects.py index c1f7b43..a6395c9 100644 --- a/ObservationSim/sim_steps/add_objects.py +++ b/ObservationSim/sim_steps/add_objects.py @@ -9,9 +9,6 @@ from ObservationSim.PSF import PSFGauss, FieldDistortion, PSFInterp, PSFInterpSL def add_objects(self, chip, filt, tel, pointing, catalog, obs_param): - # Prepare output file(s) for this chip - self.chip_output.create_output_file() - # Get exposure time if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None): exptime = obs_param["exptime"] @@ -24,6 +21,11 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param): raise ValueError("Catalog interface class must be specified for SCIE-OBS") cat = catalog(config=self.overall_config, chip=chip, pointing=pointing, chip_output=self.chip_output, filt=filt) + # Prepare output file(s) for this chip + # [NOTE] Headers of output .cat file may be updated by Catalog instance + # this should be called after the creation of Catalog instance + self.chip_output.create_output_file() + # Prepare the PSF model if self.overall_config["psf_setting"]["psf_model"] == "Gauss": psf_model = PSFGauss(chip=chip, psfRa=self.overall_config["psf_setting"]["psf_rcont"]) diff --git a/ObservationSim/sim_steps/add_pattern_noise.py b/ObservationSim/sim_steps/add_pattern_noise.py index cce62b6..4fb2598 100644 --- a/ObservationSim/sim_steps/add_pattern_noise.py +++ b/ObservationSim/sim_steps/add_pattern_noise.py @@ -5,7 +5,7 @@ from ObservationSim.Instrument.Chip import Effects def apply_PRNU(self, chip, filt, tel, pointing, catalog, obs_param): chip.img *= chip.prnu_img if self.overall_config["output_setting"]["prnu_output"] == True: - chip.prnu_img.write("%s/FlatImg_PRNU_%s.fits" % (self.chip_output.subdir, chip.chipID)) + chip.prnu_img.write("%s/FlatImg_PRNU_%s.fits" % (self.chip_output.subdir, str(chip.chipID).rjust(2, '0'))) return chip, filt, tel, pointing def add_poisson_and_dark(self, chip, filt, tel, pointing, catalog, obs_param): diff --git a/ObservationSim/sim_steps/add_sky_background.py b/ObservationSim/sim_steps/add_sky_background.py index e87eb89..8b7d67e 100644 --- a/ObservationSim/sim_steps/add_sky_background.py +++ b/ObservationSim/sim_steps/add_sky_background.py @@ -78,7 +78,7 @@ def add_sky_flat_calibration(self, chip, filt, tel, pointing, catalog, obs_param flat_normal = np.array(flat_normal, dtype='float32') if self.overall_config["output_setting"]["shutter_output"] == True: # output 16-bit shutter effect image with pixel value <=65535 shutt_gsimg = galsim.ImageUS(chip.shutter_img*6E4) - shutt_gsimg.write("%s/ShutterEffect_%s_1.fits" % (self.chip_output.subdir, chip.chipID)) + shutt_gsimg.write("%s/ShutterEffect_%s_1.fits" % (self.chip_output.subdir, str(chip.chipID).rjust(2, '0'))) self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True]) else: self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [False,'','','','']) diff --git a/config/obs_config_SCI_WIDE_phot.yaml b/config/obs_config_SCI_WIDE_phot.yaml index 18e17f9..f5613a1 100644 --- a/config/obs_config_SCI_WIDE_phot.yaml +++ b/config/obs_config_SCI_WIDE_phot.yaml @@ -14,8 +14,9 @@ obs_type_code: "101" obs_id: "00000001" # this setting will only be used if pointing list file is not given # Define list of chips -run_chips: [6,7,8,9,11,12,13,14,15,16,17,18,19,20,22,23,24,25] +# run_chips: [6,7,8,9,11,12,13,14,15,16,17,18,19,20,22,23,24,25] #run_chips: [1,2,3,4,5,10,21,26,27,28,29,30] +run_chips: [8] # Define observation sequence call_sequence: @@ -65,7 +66,7 @@ call_sequence: # Apply CCD Saturation & Blooming blooming: {} # Run CTE simulation - CTE_effect: {} + # CTE_effect: {} # Add prescan and overscan prescan_overscan: add_dark: YES diff --git a/tests/PSFInterpTest/test_PSFInterpModule_coverage.py b/tests/PSFInterpTest/test_PSFInterpModule_coverage.py index c60f1c8..6330c1e 100644 --- a/tests/PSFInterpTest/test_PSFInterpModule_coverage.py +++ b/tests/PSFInterpTest/test_PSFInterpModule_coverage.py @@ -9,10 +9,7 @@ import matplotlib as mpl mpl.use('Agg') import yaml -from ObservationSim.Config import Config -from ObservationSim.Config.Config import config_dir from ObservationSim.Instrument import Chip - from ObservationSim.PSF.PSFInterp import PSFInterp @@ -24,7 +21,6 @@ def defineCCD(iccd, config_file): # print (key + " : " + str(value)) except yaml.YAMLError as exc: print(exc) - # path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir']) chip = Chip(chipID=iccd, config=config) #chip = Chip(chipID=iccd, ccdEffCurve_dir=path_dict["ccd_dir"], CRdata_dir=path_dict["CRdata_dir"], normalize_dir=path_dict["normalize_dir"], sls_dir=path_dict['sls_dir'], config=config) return chip diff --git a/tests/PSFInterpTest/test_loadPSFSet.py b/tests/PSFInterpTest/test_loadPSFSet.py index 1059d9b..6c65c79 100644 --- a/tests/PSFInterpTest/test_loadPSFSet.py +++ b/tests/PSFInterpTest/test_loadPSFSet.py @@ -6,10 +6,7 @@ from itertools import islice import numpy as np import yaml -from ObservationSim.Config import Config -from ObservationSim.Config.Config import config_dir from ObservationSim.Instrument import Chip - from ObservationSim.PSF.PSFInterp import PSFInterp @@ -21,7 +18,6 @@ def defineCCD(iccd, config_file): # print (key + " : " + str(value)) except yaml.YAMLError as exc: print(exc) - # path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir']) chip = Chip(chipID=iccd, config=config) #chip = Chip(chipID=iccd, ccdEffCurve_dir=path_dict["ccd_dir"], CRdata_dir=path_dict["CRdata_dir"], normalize_dir=path_dict["normalize_dir"], sls_dir=path_dict['sls_dir'], config=config) return chip diff --git a/tests/TestSpecDisperse.py b/tests/TestSpecDisperse.py index 5581dd3..4929efe 100644 --- a/tests/TestSpecDisperse.py +++ b/tests/TestSpecDisperse.py @@ -1,7 +1,7 @@ import unittest from ObservationSim.MockObject.SpecDisperser import rotate90, SpecDisperser -from ObservationSim.Config import ChipOutput, Config +from ObservationSim.Config import ChipOutput from ObservationSim.Instrument import Telescope, Chip, FilterParam, Filter, FocalPlane from ObservationSim.MockObject import MockObject, Star from ObservationSim.PSF import PSFGauss @@ -401,8 +401,6 @@ class TestSpecDisperse(unittest.TestCase): print(key + " : " + str(value)) except yaml.YAMLError as exc: print(exc) - # config = Config.read_config(configFn) - # path_dict = Config.config_dir(config,work_dir=work_dir, data_dir=data_dir) filter_param = FilterParam() -- GitLab