diff --git a/Catalog/C6_50sqdeg.py b/Catalog/C6_50sqdeg.py index 8335bfa2bcf044448c532152b570fa98d19e1cb2..4ce9f3ce95089cd7377957d856bcc6efa5fa73c9 100644 --- a/Catalog/C6_50sqdeg.py +++ b/Catalog/C6_50sqdeg.py @@ -122,7 +122,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/C6_Catalog.py b/Catalog/C6_Catalog.py index 7d114e573747b84a25e892e4938d02e020e355c8..481494504d0964dc106c76960bd4d1deee3e6e6a 100644 --- a/Catalog/C6_Catalog.py +++ b/Catalog/C6_Catalog.py @@ -99,7 +99,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/C6_fits_Catalog.py b/Catalog/C6_fits_Catalog.py index 0f03dc4005bda8f5a3a32dddce742a0cbd501575..d8eea0c6b53a834dddfb3ab226e1718e5940e727 100644 --- a/Catalog/C6_fits_Catalog.py +++ b/Catalog/C6_fits_Catalog.py @@ -113,7 +113,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/Calib_Catalog.py b/Catalog/Calib_Catalog.py index 3987ef548097419e2ca2889739e8e2134211d9ee..308a9d5fd08146b1e266ba41f692cb760f9395b4 100644 --- a/Catalog/Calib_Catalog.py +++ b/Catalog/Calib_Catalog.py @@ -106,7 +106,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/FGS_Catalog.py b/Catalog/FGS_Catalog.py index 3b0c23024d87a7e2cc8074692a3a401657f78e55..bdab18cd497a1b5b72a719961c83557cfe2db225 100644 --- a/Catalog/FGS_Catalog.py +++ b/Catalog/FGS_Catalog.py @@ -100,7 +100,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/NGPCatalog.py b/Catalog/NGPCatalog.py index f37b73c467297ba5f514e2d374640b25b5626175..cf6ae97a2758d98ab827f5e94f06222c5a9cdd0b 100644 --- a/Catalog/NGPCatalog.py +++ b/Catalog/NGPCatalog.py @@ -65,7 +65,7 @@ class Catalog(CatalogBase): def _add_output_columns_header(self): self.add_hdr = " model_tag teff logg feh" self.add_fmt = " %10s %8.4f %8.4f %8.4f" - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/fd_test_C6.py b/Catalog/fd_test_C6.py index 0b7909a3e3c7dfe3dce5cc2c3d0f55e66a9011a9..cbb41e88bcdeed54074a4ef2bca09f053ed1e31c 100644 --- a/Catalog/fd_test_C6.py +++ b/Catalog/fd_test_C6.py @@ -99,7 +99,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/testCat_fits.py b/Catalog/testCat_fits.py index 2c73c507af97630c08ed45ef0e4b8940dfec2ab3..b148431b8489c92357fce6798596d63ba3c5c75d 100644 --- a/Catalog/testCat_fits.py +++ b/Catalog/testCat_fits.py @@ -77,7 +77,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/testCat_galaxy.py b/Catalog/testCat_galaxy.py index 8ecd698e421cadd9aeb742701cab191d701776c8..fd9a5a41c686ebee91ab5633e760dc6902325551 100644 --- a/Catalog/testCat_galaxy.py +++ b/Catalog/testCat_galaxy.py @@ -97,7 +97,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/testCat_star.py b/Catalog/testCat_star.py index 50207daac87b111d7cf4097f02a3b39270377ce9..458e41cd9756c974204633bf4c1ea454726603c5 100644 --- a/Catalog/testCat_star.py +++ b/Catalog/testCat_star.py @@ -124,7 +124,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/Catalog/wcs_test_C6.py b/Catalog/wcs_test_C6.py index 896ab4ae5e4cd86a861509dc5556f709c7769e12..5623ac23e71c4795dea569959058aa9957a2b042 100644 --- a/Catalog/wcs_test_C6.py +++ b/Catalog/wcs_test_C6.py @@ -99,7 +99,7 @@ class Catalog(CatalogBase): self.add_fmt = " %10s %8.4f %8.4f %8.4f" self.add_fmt += " %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %8.4f %4d %8.4f " - self.chip_output.update_ouptut_header(additional_column_names=self.add_hdr) + self.chip_output.update_output_header(additional_column_names=self.add_hdr) def _get_healpix_list(self): self.sky_coverage = self.chip.getSkyCoverageEnlarged(self.chip.img.wcs, margin=0.2) diff --git a/ObservationSim/Config/ChipOutput.py b/ObservationSim/Config/ChipOutput.py index 4ad076243288dd59e9f1f43b6f3a8081fb860e43..5d11380f013a704fe308284876388112d3a15fe8 100755 --- a/ObservationSim/Config/ChipOutput.py +++ b/ObservationSim/Config/ChipOutput.py @@ -1,35 +1,40 @@ import os import logging import ObservationSim.Config._util as _util +from ObservationSim.Config.Header import generatePrimaryHeader class ChipOutput(object): - def __init__(self, config, focal_plane, chip, filt, imgKey0="", imgKey1="", imgKey2="", exptime=150., mjdTime="", ra_cen=None, dec_cen=None, pointing_type='SCI', pointing_ID='0',pointing_type_code='101', subdir="./", prefix=""): - self.focal_plane = focal_plane + def __init__(self, config, chip, filt, pointing): + self.config = config self.chip = chip self.filt = filt - self.imgKey0 = imgKey0 - self.imgKey1 = imgKey1 - self.imgKey2 = imgKey2 - self.exptime = exptime - self.mjdTime = mjdTime - self.pointing_type = pointing_type - if (ra_cen is not None) and (dec_cen is not None): - self.ra_cen = ra_cen - self.dec_cen = dec_cen - else: - self.ra_cen = config["obs_setting"]["ra_center"] - self.dec_cen = config["obs_setting"]["dec_center"] + self.pointing_type = pointing.pointing_type + self.chip_label = str(chip.chipID).rjust(2, '0') - self.chipLabel = focal_plane.getChipLabel(chip.chipID) - obs_id = _util.get_obs_id(img_type=pointing_type, project_cycle=config["project_cycle"], run_counter=config["run_counter"], pointing_id=pointing_ID, pointing_type_code = pointing_type_code) - # self.cat_name = "MSC_1%s_chip_%s_filt_%s"%(str(pointing_ID).rjust(8, '0'), self.chipLabel, filt.filter_type) + ".cat" - self.cat_name = "MSC_%s_chip_%s_filt_%s"%(obs_id, self.chipLabel, filt.filter_type) + ".cat" + # Get primary header based on chip and pointing + self.h_prim = generatePrimaryHeader( + xlen=chip.npix_x, + ylen=chip.npix_y, + pointing_id = pointing.obs_id, + pointing_type_code = pointing.pointing_type_code, + ra=pointing.ra, + dec=pointing.dec, + pixel_scale=chip.pix_scale, + time_pt = pointing.timestamp, + exptime=pointing.exp_time, + im_type=pointing.pointing_type, + sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z], + sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz], + project_cycle=self.config["project_cycle"], + run_counter=self.config["run_counter"], + chip_name=self.chip_label) - self.subdir = subdir + obs_id = _util.get_obs_id(img_type=self.pointing_type, project_cycle=config["project_cycle"], run_counter=config["run_counter"], pointing_id=pointing.obs_id, pointing_type_code = pointing.pointing_type_code) + + self.subdir = pointing.output_dir + self.cat_name = self.h_prim['FILENAME'] + '.cat' + logger_filename = self.h_prim['FILENAME'] + '.log' - # Setup logger for each chip - # logger_filename = "MSC_1%s_chip_%s_filt_%s"%(str(pointing_ID).rjust(8, '0'), self.chipLabel, filt.filter_type) + ".log" - logger_filename = "MSC_%s_chip_%s_filt_%s"%(obs_id, self.chipLabel, filt.filter_type) + ".log" self.logger = logging.getLogger() fh = logging.FileHandler(os.path.join(self.subdir, logger_filename), mode='w+', encoding='utf-8') fh.setLevel(logging.DEBUG) @@ -46,7 +51,7 @@ class ChipOutput(object): self.hdr = hdr1 + hdr2 self.fmt = fmt1 + fmt2 - self.logger.info("pointing_type = %s\n"%(pointing_type)) + self.logger.info("pointing_type = %s\n"%(self.pointing_type)) def Log_info(self, message): print(message) @@ -56,7 +61,7 @@ class ChipOutput(object): print(message) self.logger.error(message) - def update_ouptut_header(self, additional_column_names=""): + def update_output_header(self, additional_column_names=""): self.hdr += additional_column_names def create_output_file(self): @@ -68,14 +73,11 @@ class ChipOutput(object): self.cat.write(self.hdr) def cat_add_obj(self, obj, pos_img, pos_shear): - # self.logger.info('-------------debug-----------------') - # self.logger.info('from global',ximg, yimg) ximg = obj.real_pos.x + 1.0 yimg = obj.real_pos.y + 1.0 - # self.logger.info('from loacl',ximg, yimg) line = self.fmt%( - obj.id, int(self.chipLabel), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.ra_orig, obj.dec_orig, obj.z, obj.getMagFilter(self.filt), obj.type, + obj.id, int(self.chip_label), self.filt.filter_type, ximg, yimg, obj.ra, obj.dec, obj.ra_orig, obj.dec_orig, obj.z, obj.getMagFilter(self.filt), obj.type, obj.pmra, obj.pmdec, obj.rv, obj.parallax) line += obj.additional_output_str if not line.endswith("\n"): diff --git a/ObservationSim/Config/Config.py b/ObservationSim/Config/Config.py deleted file mode 100755 index 85120265fc3529d0eb115b428e465a106d8a8904..0000000000000000000000000000000000000000 --- a/ObservationSim/Config/Config.py +++ /dev/null @@ -1,93 +0,0 @@ -import galsim -import os -from astropy.time import Time as asTime - -def config_dir(config, work_dir=None, data_dir=None): - path_dict = {} - # Working directory - if work_dir == None: - dirname, _ = os.path.split(os.path.abspath(__file__)) - path_dict["work_dir"] = "/".join(dirname.split("/")[:-1]) + "/" - else: - path_dict["work_dir"] = work_dir - - # Data directory - # if data_dir == None: - # # Assume all input datasets are in the work directory - # path_dict["data_dir"] =os.path.join(path_dict["work_dir"], "data/") - # else: - # path_dict["data_dir"] = data_dir - - # PSF data directory - # if config["psf_setting"]["psf_model"] == "Interp": - # path_dict["psf_pho_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_pho_dir"]) - # path_dict["psf_sls_dir"] = os.path.join(path_dict["data_dir"], config["psf_setting"]["psf_sls_dir"]) - if config["psf_setting"]["psf_model"] == "Interp": - path_dict["psf_pho_dir"] = config["psf_setting"]["psf_pho_dir"] - path_dict["psf_sls_dir"] = config["psf_setting"]["psf_sls_dir"] - - return path_dict - -def read_config(config_filename): - """Read in a configuration file and return the corresponding dict(s). - - Parameters: - config_filename: The name of the configuration file to read. - Returns: - (list) of config dicts - """ - config = {} - config_file = open(config_filename).readlines() - nlines = len(config_file) - for i in range(nlines): - row = config_file[i].split() - if len(row) <= 1: continue # blank row - if not "#" in row: - if len(row) == 2: - key, val = row[0:2] - config.update({key:val}) - else: - print("!! Something is wrong with parameter '%s'."%row[0]) - return - elif row.index("#") == 2: - key, val = row[0:2] - config.update({key:val}) - elif row.index("#") == 0: - continue # annotation - else: - print("!! Something is wrong with parameter '%s'."%row[0]) - return - config = parse_config(config) - return config - -def parse_config(config): - """Parse the config values to the right type - - Parameters: - config: raw config dict - Returns: - Parsed config dict - """ - config["ra_center"] = float(config["ra_center"]) - config["dec_center"] = float(config["dec_center"]) - config["psf_rcont"] = config["psf_rcont"].split(",") - config["psfRa"] = float(config["psf_rcont"][0]) - config["psfCont"] = float(config["psf_rcont"][1]) - config["image_rot"] = float(config["image_rot"])*galsim.degrees - config["sigma_spin"] = float(config["sigma_spin"]) - config["reduced_g1"] = float(config["reduced_g1"]) - config["reduced_g2"] = float(config["reduced_g2"]) - config["rotateEll"] = float(config["rotateEll"]) - config["reEll"] = int(config["rotateEll"]/45.0) - if config["reEll"]==0: config["reIndex"] = "P" - if config["reEll"]==1: config["reIndex"] = "X" - if config["reEll"]==2: config["reIndex"] = "N" - if config["reEll"]==3: config["reIndex"] = "Y" - config["seed_flat"] = int(config["seed_flat"]) - config["seed_prnu"] = int(config["seed_prnu"]) - config["seed_star"] = int(config["seed_star"]) - config["seed_gal"] = int(config["seed_gal"]) - config["seed_Av"] = int(config["seed_Av"]) - config["bias_level"] = int(config["bias_level"]) - config["df_strength"] = float(config["df_strength"]) - return config diff --git a/ObservationSim/Config/Pointing.py b/ObservationSim/Config/Pointing.py index 9eb0f1ab5ca7170db54f65e64b7d6bbceed98433..6d95b4d5d3a0a9cc88d4f92a38584aa4dcf3e116 100644 --- a/ObservationSim/Config/Pointing.py +++ b/ObservationSim/Config/Pointing.py @@ -27,6 +27,7 @@ class Pointing(object): self.obs_config_file = obs_config_file self.t_shutter_open = t_shutter_open self.t_shutter_close = t_shutter_close + self.output_dir = "." if self.obs_config_file is not None: with open(self.obs_config_file, "r") as stream: try: @@ -107,7 +108,7 @@ class Pointing(object): else: self.timestamp = t - def make_output_pointing_dir(self, overall_config, pointing_ID=0, copy_obs_config=False): + def make_output_pointing_dir(self, overall_config, copy_obs_config=False): run_dir = os.path.join(overall_config["work_dir"], overall_config["run_name"]) if not os.path.exists(run_dir): try: @@ -131,7 +132,7 @@ class Pointing(object): obs_config_output_path = os.path.join(self.output_dir, os.path.basename(self.obs_config_file)) if not os.path.exists(obs_config_output_path): try: - shutil.copy(args.obs_config_file, self.output_dir) + shutil.copy(self.obs_config_file, self.output_dir) except OSError: pass diff --git a/ObservationSim/Config/__init__.py b/ObservationSim/Config/__init__.py index 96ca93e25999a633ba3697782b2d72e7e66b17fa..bf03e61f5078114b03a38a19d2b43fae72b13341 100644 --- a/ObservationSim/Config/__init__.py +++ b/ObservationSim/Config/__init__.py @@ -1,3 +1,2 @@ -from .Config import * from .ChipOutput import ChipOutput from .Pointing import Pointing \ No newline at end of file diff --git a/ObservationSim/ObservationSim.py b/ObservationSim/ObservationSim.py index 3421713f47bd8d7b1b2cd71ba187fede7079b108..3d8030de425051b32de3b5d31bc273ecee3a4874 100755 --- a/ObservationSim/ObservationSim.py +++ b/ObservationSim/ObservationSim.py @@ -8,17 +8,15 @@ from datetime import datetime import traceback -from ObservationSim.Config import config_dir, ChipOutput +from ObservationSim.Config import ChipOutput from ObservationSim.Instrument import Telescope, Filter, FilterParam, FocalPlane, Chip from ObservationSim.Instrument.Chip import Effects from ObservationSim.Instrument.Chip import ChipUtils as chip_utils -from ObservationSim._util import make_output_pointing_dir from ObservationSim.Astrometry.Astrometry_util import on_orbit_obs_position from ObservationSim.sim_steps import SimSteps, SIM_STEP_TYPES class Observation(object): def __init__(self, config, Catalog, work_dir=None, data_dir=None): - self.path_dict = config_dir(config=config, work_dir=work_dir, data_dir=data_dir) self.config = config self.tel = Telescope() self.filter_param = FilterParam() @@ -92,9 +90,6 @@ class Observation(object): # Prepare necessary chip properties for simulation chip = self.prepare_chip_for_exposure(chip, ra_cen, dec_cen, pointing) - # Load catalogues - # self.cat = self.Catalog(config=self.config, chip=chip, pointing=pointing, cat_dir=cat_dir, sed_dir=sed_dir, chip_output=chip_output, filt=filt) - # Initialize SimSteps sim_steps = SimSteps(overall_config=self.config, chip_output=chip_output, all_filters=self.all_filters) @@ -124,138 +119,6 @@ class Observation(object): chip_output.Log_info("check running:1: pointing-%d chip-%d pid-%d memory-%6.2fGB"%(pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024) )) del chip.img - # def run_one_chip_calibration(self, chip, filt, pointing, chip_output, skyback_level = 20000, sky_level_filt = 'g', wcs_fp=None, psf_model=None, cat_dir=None, sed_dir=None): - - - - - # # # Get WCS for the focal plane - # # if wcs_fp == None: - # # wcs_fp = self.focal_plane.getTanWCS(ra_cen, dec_cen, pointing.img_pa, chip.pix_scale) - - # # Create chip Image - # chip.img = galsim.ImageF(chip.npix_x, chip.npix_y) - # chip.img.setOrigin(chip.bound.xmin, chip.bound.ymin) - # # chip.img.wcs = wcs_fp - # pf_map = np.zeros_like(chip.img.array) - # if self.config["obs_setting"]["LED_TYPE"] is not None: - # if len(self.config["obs_setting"]["LED_TYPE"]) != 0: - # print("LED OPEN--------") - - # led_obj = FlatLED(chip, filt) - - # led_flat = led_obj.drawObj_LEDFlat(led_type_list=self.config["obs_setting"]["LED_TYPE"], exp_t_list=self.config["obs_setting"]["LED_TIME"]) - # pf_map = led_flat - # # whether to output zero, dark, flat calibration images. - # expTime = self.config["obs_setting"]["exp_time"] - # norm_scaler = skyback_level/expTime / self.filter_param.param[sky_level_filt][5] - - # if skyback_level == 0: - # self.config["ins_effects"]["shutter_effect"] = False - - # if chip.survey_type == "photometric": - # sky_map = np.ones_like(chip.img.array) * norm_scaler * self.filter_param.param[chip.filter_type][5] / self.tel.pupil_area - # elif chip.survey_type == "spectroscopic": - # flat_normal = np.ones_like(chip.img.array) - # if self.config["ins_effects"]["flat_fielding"] == True: - # chip_output.Log_info("SLS flat preprocess,CHIP %d : Creating and applying Flat-Fielding" % chip.chipID) - # msg = str(chip.img.bounds) - # chip_output.Log_info(msg) - # flat_img = Effects.MakeFlatSmooth( - # chip.img.bounds, - # int(self.config["random_seeds"]["seed_flat"])) - # flat_normal = flat_normal * flat_img.array / np.mean(flat_img.array) - # if self.config["ins_effects"]["shutter_effect"] == True: - # chip_output.Log_info("SLS flat preprocess,CHIP %d : Apply shutter effect" % chip.chipID) - # shuttimg = Effects.ShutterEffectArr(chip.img, t_shutter=1.3, dist_bearing=735, - # dt=1E-3) # shutter effect normalized image for this chip - # flat_normal = flat_normal * shuttimg - # flat_normal = np.array(flat_normal, dtype='float32') - # sky_map = calculateSkyMap_split_g( - # skyMap=flat_normal, - # blueLimit=filt.blue_limit, - # redLimit=filt.red_limit, - # conf=chip.sls_conf, - # pixelSize=chip.pix_scale, - # isAlongY=0, - # flat_cube=chip.flat_cube) - # sky_map = sky_map * norm_scaler - - # chip.img = chip.addEffects( - # config=self.config, - # img=chip.img, - # chip_output=chip_output, - # filt=filt, - # ra_cen=pointing.ra, - # dec_cen=pointing.dec, - # img_rot=pointing.img_pa, - # exptime=self.config["obs_setting"]["exp_time"], - # pointing_ID=pointing.id, - # timestamp_obs=pointing.timestamp, - # pointing_type=pointing.pointing_type, - # sky_map=sky_map, tel=self.tel, - # post_flash_map=pf_map, - # logger=chip_output.logger) - - # datetime_obs = datetime.utcfromtimestamp(pointing.timestamp) - # date_obs = datetime_obs.strftime("%y%m%d") - # time_obs = datetime_obs.strftime("%H%M%S") - # h_prim = generatePrimaryHeader( - # xlen=chip.npix_x, - # ylen=chip.npix_y, - # pointNum=str(pointing.id), - # ra=pointing.ra, - # dec=pointing.dec, - # pixel_scale=chip.pix_scale, - # date=date_obs, - # time_obs=time_obs, - # exptime=self.config["obs_setting"]["exp_time"], - # im_type='DARKPF', - # sat_pos=[pointing.sat_x, pointing.sat_y, pointing.sat_z], - # sat_vel=[pointing.sat_vx, pointing.sat_vy, pointing.sat_vz], - # chip_name=str(chip.chipID).rjust(2, '0')) - # h_ext = generateExtensionHeader( - # chip=chip, - # xlen=chip.npix_x, - # ylen=chip.npix_y, - # ra=pointing.ra, - # dec=pointing.dec, - # pa=pointing.img_pa.deg, - # gain=chip.gain, - # readout=chip.read_noise, - # dark=chip.dark_noise, - # saturation=90000, - # pixel_scale=chip.pix_scale, - # pixel_size=chip.pix_size, - # xcen=chip.x_cen, - # ycen=chip.y_cen, - # extName='SCI', - # timestamp=pointing.timestamp, - # exptime=self.config["obs_setting"]["exp_time"], - # readoutTime=chip.readout_time) - - # chip.img = galsim.Image(chip.img.array, dtype=np.uint16) - # hdu1 = fits.PrimaryHDU(header=h_prim) - # hdu1.add_checksum() - # hdu1.header.comments['CHECKSUM'] = 'HDU checksum' - # hdu1.header.comments['DATASUM'] = 'data unit checksum' - # hdu2 = fits.ImageHDU(chip.img.array, header=h_ext) - # hdu2.add_checksum() - # hdu2.header.comments['XTENSION'] = 'extension type' - # hdu2.header.comments['CHECKSUM'] = 'HDU checksum' - # hdu2.header.comments['DATASUM'] = 'data unit checksum' - # hdu1 = fits.HDUList([hdu1, hdu2]) - # fname = os.path.join(chip_output.subdir, h_prim['FILENAME'] + '.fits') - # hdu1.writeto(fname, output_verify='ignore', overwrite=True) - - # # chip_output.Log_info("# objects that are too bright %d out of %d" % (bright_obj, self.nobj)) - # # chip_output.Log_info("# objects that are too dim %d out of %d" % (dim_obj, self.nobj)) - # # chip_output.Log_info("# objects that are missed %d out of %d" % (missed_obj, self.nobj)) - # del chip.img - - # chip_output.Log_info("check running:2: pointing-%d chip-%d pid-%d memory-%6.2fGB" % ( - # pointing.id, chip.chipID, os.getpid(), (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024 / 1024))) - def runExposure_MPI_PointingList(self, pointing_list, chips=None, use_mpi=False): if use_mpi: comm = MPI.COMM_WORLD @@ -266,9 +129,10 @@ class Observation(object): for ipoint in range(len(pointing_list)): # Construct chips & filters: pointing = pointing_list[ipoint] - pointing_ID = pointing.id + # pointing_ID = pointing.id + pointing_ID = pointing.obs_id - sub_img_dir, prefix = make_output_pointing_dir(path_dict=self.path_dict, config=self.config, pointing_ID=pointing_ID) + pointing.make_output_pointing_dir(overall_config=self.config, copy_obs_config=True) self.focal_plane = FocalPlane(chip_list=pointing.obs_param["run_chips"]) # Make Chip & Filter lists @@ -314,39 +178,19 @@ class Observation(object): chip = run_chips[ichip] filt = run_filts[ichip] - # chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid)) + chip_output = ChipOutput( - config=self.config, - focal_plane=self.focal_plane, - chip=chip, - filt=filt, - exptime=pointing.exp_time, - pointing_type=pointing.pointing_type, - pointing_ID=pointing.obs_id, - pointing_type_code = pointing.pointing_type_code, - subdir=sub_img_dir, - prefix=prefix, - ra_cen=pointing.ra, - dec_cen=pointing.dec) - chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(pointing_ID, chip.chipID, pid)) + config = self.config, + chip = chip, + filt = filt, + pointing = pointing + ) + chip_output.Log_info("running pointing#%d, chip#%d, at PID#%d..."%(int(pointing_ID), chip.chipID, pid)) self.run_one_chip( chip=chip, filt=filt, chip_output=chip_output, pointing=pointing) - # if self.config["obs_setting"]["survey_type"] == "CALIBRATION": - # self.run_one_chip_calibration(chip=chip, - # filt=filt, - # chip_output=chip_output, - # pointing=pointing, - # skyback_level = self.config["obs_setting"]["FLAT_LEVEL"], - # sky_level_filt = self.config["obs_setting"]["FLAT_LEVEL_FIL"]) - # else: - # self.run_one_chip( - # chip=chip, - # filt=filt, - # chip_output=chip_output, - # pointing=pointing) chip_output.Log_info("finished running chip#%d..."%(chip.chipID)) for handler in chip_output.logger.handlers[:]: chip_output.logger.removeHandler(handler) diff --git a/ObservationSim/sim_steps/add_objects.py b/ObservationSim/sim_steps/add_objects.py index c1f7b4375c74f5f4fb6343dded4a4b247c12578f..a6395c90adac68bb47deb8ed42208c206dd87b76 100644 --- a/ObservationSim/sim_steps/add_objects.py +++ b/ObservationSim/sim_steps/add_objects.py @@ -9,9 +9,6 @@ from ObservationSim.PSF import PSFGauss, FieldDistortion, PSFInterp, PSFInterpSL def add_objects(self, chip, filt, tel, pointing, catalog, obs_param): - # Prepare output file(s) for this chip - self.chip_output.create_output_file() - # Get exposure time if (obs_param) and ("exptime" in obs_param) and (obs_param["exptime"] is not None): exptime = obs_param["exptime"] @@ -24,6 +21,11 @@ def add_objects(self, chip, filt, tel, pointing, catalog, obs_param): raise ValueError("Catalog interface class must be specified for SCIE-OBS") cat = catalog(config=self.overall_config, chip=chip, pointing=pointing, chip_output=self.chip_output, filt=filt) + # Prepare output file(s) for this chip + # [NOTE] Headers of output .cat file may be updated by Catalog instance + # this should be called after the creation of Catalog instance + self.chip_output.create_output_file() + # Prepare the PSF model if self.overall_config["psf_setting"]["psf_model"] == "Gauss": psf_model = PSFGauss(chip=chip, psfRa=self.overall_config["psf_setting"]["psf_rcont"]) diff --git a/ObservationSim/sim_steps/add_pattern_noise.py b/ObservationSim/sim_steps/add_pattern_noise.py index cce62b6da993d60008860e1aa1b498affc970366..4fb2598ae15062d766d34207924497416abbd767 100644 --- a/ObservationSim/sim_steps/add_pattern_noise.py +++ b/ObservationSim/sim_steps/add_pattern_noise.py @@ -5,7 +5,7 @@ from ObservationSim.Instrument.Chip import Effects def apply_PRNU(self, chip, filt, tel, pointing, catalog, obs_param): chip.img *= chip.prnu_img if self.overall_config["output_setting"]["prnu_output"] == True: - chip.prnu_img.write("%s/FlatImg_PRNU_%s.fits" % (self.chip_output.subdir, chip.chipID)) + chip.prnu_img.write("%s/FlatImg_PRNU_%s.fits" % (self.chip_output.subdir, str(chip.chipID).rjust(2, '0'))) return chip, filt, tel, pointing def add_poisson_and_dark(self, chip, filt, tel, pointing, catalog, obs_param): diff --git a/ObservationSim/sim_steps/add_sky_background.py b/ObservationSim/sim_steps/add_sky_background.py index e87eb8934a05a1e077868d52d1a4ff439ec5cf9b..8b7d67e6f48d9e48aac46914676ef7a934231d8f 100644 --- a/ObservationSim/sim_steps/add_sky_background.py +++ b/ObservationSim/sim_steps/add_sky_background.py @@ -78,7 +78,7 @@ def add_sky_flat_calibration(self, chip, filt, tel, pointing, catalog, obs_param flat_normal = np.array(flat_normal, dtype='float32') if self.overall_config["output_setting"]["shutter_output"] == True: # output 16-bit shutter effect image with pixel value <=65535 shutt_gsimg = galsim.ImageUS(chip.shutter_img*6E4) - shutt_gsimg.write("%s/ShutterEffect_%s_1.fits" % (self.chip_output.subdir, chip.chipID)) + shutt_gsimg.write("%s/ShutterEffect_%s_1.fits" % (self.chip_output.subdir, str(chip.chipID).rjust(2, '0'))) self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT'], values = [True]) else: self.updateHeaderInfo(header_flag='ext', keys = ['SHTSTAT','SHTOPEN0','SHTOPEN1','SHTCLOS0','SHTCLOS1'], values = [False,'','','','']) diff --git a/config/obs_config_SCI_WIDE_phot.yaml b/config/obs_config_SCI_WIDE_phot.yaml index 18e17f9f7f1e8b9e767784e71479005d6fffb2c1..f5613a10d293f914e8fc86eb88910178084b0b2f 100644 --- a/config/obs_config_SCI_WIDE_phot.yaml +++ b/config/obs_config_SCI_WIDE_phot.yaml @@ -14,8 +14,9 @@ obs_type_code: "101" obs_id: "00000001" # this setting will only be used if pointing list file is not given # Define list of chips -run_chips: [6,7,8,9,11,12,13,14,15,16,17,18,19,20,22,23,24,25] +# run_chips: [6,7,8,9,11,12,13,14,15,16,17,18,19,20,22,23,24,25] #run_chips: [1,2,3,4,5,10,21,26,27,28,29,30] +run_chips: [8] # Define observation sequence call_sequence: @@ -65,7 +66,7 @@ call_sequence: # Apply CCD Saturation & Blooming blooming: {} # Run CTE simulation - CTE_effect: {} + # CTE_effect: {} # Add prescan and overscan prescan_overscan: add_dark: YES diff --git a/tests/PSFInterpTest/test_PSFInterpModule_coverage.py b/tests/PSFInterpTest/test_PSFInterpModule_coverage.py index c60f1c857ebde368d2774d6c06b891ca57a2aeb4..6330c1efe1437e71717e9026d0420c3114718311 100644 --- a/tests/PSFInterpTest/test_PSFInterpModule_coverage.py +++ b/tests/PSFInterpTest/test_PSFInterpModule_coverage.py @@ -9,10 +9,7 @@ import matplotlib as mpl mpl.use('Agg') import yaml -from ObservationSim.Config import Config -from ObservationSim.Config.Config import config_dir from ObservationSim.Instrument import Chip - from ObservationSim.PSF.PSFInterp import PSFInterp @@ -24,7 +21,6 @@ def defineCCD(iccd, config_file): # print (key + " : " + str(value)) except yaml.YAMLError as exc: print(exc) - # path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir']) chip = Chip(chipID=iccd, config=config) #chip = Chip(chipID=iccd, ccdEffCurve_dir=path_dict["ccd_dir"], CRdata_dir=path_dict["CRdata_dir"], normalize_dir=path_dict["normalize_dir"], sls_dir=path_dict['sls_dir'], config=config) return chip diff --git a/tests/PSFInterpTest/test_loadPSFSet.py b/tests/PSFInterpTest/test_loadPSFSet.py index 1059d9b0bb8f046e9c36e3aa273ea130be645abe..6c65c79d458802f6f61fb90d0017dd2efe67f031 100644 --- a/tests/PSFInterpTest/test_loadPSFSet.py +++ b/tests/PSFInterpTest/test_loadPSFSet.py @@ -6,10 +6,7 @@ from itertools import islice import numpy as np import yaml -from ObservationSim.Config import Config -from ObservationSim.Config.Config import config_dir from ObservationSim.Instrument import Chip - from ObservationSim.PSF.PSFInterp import PSFInterp @@ -21,7 +18,6 @@ def defineCCD(iccd, config_file): # print (key + " : " + str(value)) except yaml.YAMLError as exc: print(exc) - # path_dict = config_dir(config=config, work_dir=config['work_dir'], data_dir=config['data_dir']) chip = Chip(chipID=iccd, config=config) #chip = Chip(chipID=iccd, ccdEffCurve_dir=path_dict["ccd_dir"], CRdata_dir=path_dict["CRdata_dir"], normalize_dir=path_dict["normalize_dir"], sls_dir=path_dict['sls_dir'], config=config) return chip diff --git a/tests/TestSpecDisperse.py b/tests/TestSpecDisperse.py index 5581dd3e62cc63daebb4408df82513167fb1e473..4929efe599da14ef7938d722e1c90c80b086e801 100644 --- a/tests/TestSpecDisperse.py +++ b/tests/TestSpecDisperse.py @@ -1,7 +1,7 @@ import unittest from ObservationSim.MockObject.SpecDisperser import rotate90, SpecDisperser -from ObservationSim.Config import ChipOutput, Config +from ObservationSim.Config import ChipOutput from ObservationSim.Instrument import Telescope, Chip, FilterParam, Filter, FocalPlane from ObservationSim.MockObject import MockObject, Star from ObservationSim.PSF import PSFGauss @@ -401,8 +401,6 @@ class TestSpecDisperse(unittest.TestCase): print(key + " : " + str(value)) except yaml.YAMLError as exc: print(exc) - # config = Config.read_config(configFn) - # path_dict = Config.config_dir(config,work_dir=work_dir, data_dir=data_dir) filter_param = FilterParam()