diff --git a/tools/setConfig/.DS_Store b/tools/setConfig/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..316c88df0f1de73f1e37a841b6e985fdc1b95458 Binary files /dev/null and b/tools/setConfig/.DS_Store differ diff --git a/tools/setConfig/reset_obs.py b/tools/setConfig/reset_obs.py new file mode 100644 index 0000000000000000000000000000000000000000..c8c7a65133df5b993edb95c3a344c37503659b8e --- /dev/null +++ b/tools/setConfig/reset_obs.py @@ -0,0 +1,118 @@ +from flask import Flask, render_template, request, redirect +import yaml +import ast + +app = Flask(__name__) + + +key_type_map = { + 'obs_type': str, + 'obs_type_code': str, + 'obs_id': str, + 'run_chips': list, + 'call_sequence':{ + 'scie_obs':{ + 'shutter_effect': bool, + 'flat_fielding': bool, + 'field_dist': bool, + }, + 'sky_background':{ + 'shutter_effect': bool, + 'flat_fielding': bool, + 'enable_straylight_model': bool, + 'flat_level': None, + 'flat_level_filt': None, + }, + 'PRNU_effect': {}, + 'cosmic_rays':{ + 'save_cosmic_img': bool, + }, + 'poisson_and_dark':{ + 'add_dark': bool, + }, + 'bright_fatter': {}, + 'detector_defects':{ + 'hot_pixels': bool, + 'dead_pixels': bool, + 'bad_columns': bool, + }, + 'nonlinearity': {}, + 'blooming': {}, + 'prescan_overscan':{ + 'add_dark': bool, + }, + 'bias':{ + 'bias_16channel': bool, + }, + 'readout_noise': {}, + 'gain':{ + 'gain_16channel': bool, + }, + 'quantization_and_output':{ + 'format_output': bool, + }, + }, +} +def convert_dict_values(d, key_type_map): + for key, value in d.items(): + if isinstance(value, dict): + convert_dict_values(value, key_type_map[key]) + elif key in key_type_map: + if key_type_map[key] is int: + d[key] = int(value) + if key_type_map[key] is float: + d[key] = float(value) + if key_type_map[key] is bool: + if d[key].lower() == 'yes' or d[key].lower() == 'true': + d[key] = True + else: + d[key] = False + if key_type_map[key] is str: + d[key] = str(value) + if key_type_map[key] is list: + d[key] = ast.literal_eval(value) + if key_type_map[key] is None: + d[key] = None + + + +def load_yaml(): + with open('templates/obs_config_SCI.yaml', 'r') as file: + return yaml.safe_load(file) + +def save_yaml(data): + convert_dict_values(data, key_type_map) + with open('config_reset/obs_config_SCI_reset.yaml', 'w') as file: + yaml.dump(data, file, default_flow_style=False, sort_keys=False) + +def render_form(data, parent_key=''): + form_html = '' + for key, value in data.items(): + full_key = f"{parent_key}.{key}" if parent_key else key + if isinstance(value, dict): # 处理字典 + form_html += f"

{key}

{render_form(value, full_key)}
" + else: + form_html += f"" + form_html += f"
" + return form_html + +@app.route('/', methods=['GET', 'POST']) +def index(): + if request.method == 'POST': + data = load_yaml() + for key in request.form: + keys = key.split('.') + temp = data + for k in keys[:-1]: + temp = temp[k] + temp[keys[-1]] = request.form[key] + save_yaml(data) + return redirect('/') + + data = load_yaml() + form_html = render_form(data) + return render_template('index_obs.html', form_html=form_html) + +if __name__ == '__main__': + app.run(debug=True) + diff --git a/tools/setConfig/reset_overall.py b/tools/setConfig/reset_overall.py new file mode 100644 index 0000000000000000000000000000000000000000..df51b24dbafb8e5fe0849a357c48518624517776 --- /dev/null +++ b/tools/setConfig/reset_overall.py @@ -0,0 +1,127 @@ +from flask import Flask, render_template, request, redirect +import yaml +import ast + +app = Flask(__name__) + +key_type_map = { + 'work_dir': str, + 'run_name': str, + 'project_cycle': int, + 'run_counter': int, + 'run_option':{ + 'out_cat_only': bool, + }, + 'catalog_options':{ + 'input_path':{ + 'cat_dir': str, + 'star_cat': str, + 'galaxy_cat': str, + }, + 'SED_templates_path':{ + 'star_SED': str, + 'galaxy_SED': str, + 'AGN_SED': str, + }, + 'star_only': bool, + 'galaxy_only': bool, + 'rotateEll': float, + 'enable_mw_ext_gal': bool, + 'planck_ebv_map':str, + }, + 'obs_setting':{ + 'pointing_file': str, + 'obs_config_file': str, + 'run_pointings': list, + 'enable_astrometric_model': bool, + 'cut_in_band': str, + 'mag_sat_margin': float, + 'mag_lim_margin': float, + }, + 'psf_setting':{ + 'psf_model': str, + 'psf_pho_dir': str, + 'psf_sls_dir': str, + }, + 'shear_setting':{ + 'shear_type': str, + 'reduced_g1': float, + 'reduced_g2': float, + }, + 'output_setting':{ + 'output_format': str, + 'shutter_output': bool, + 'prnu_output': bool, + }, + 'random_seeds':{ + 'seed_poisson': int, + 'seed_CR': int, + 'seed_flat': int, + 'seed_prnu': int, + 'seed_gainNonUniform': int, + 'seed_biasNonUniform': int, + 'seed_rnNonUniform': int, + 'seed_badcolumns': int, + 'seed_defective': int, + 'seed_readout': int, + }, +} +def convert_dict_values(d, key_type_map): + for key, value in d.items(): + if isinstance(value, dict): + convert_dict_values(value, key_type_map[key]) + elif key in key_type_map: + if key_type_map[key] is int: + d[key] = int(value) + if key_type_map[key] is float: + d[key] = float(value) + if key_type_map[key] is bool: + if d[key].lower() == 'yes' or d[key].lower() == 'true': + d[key] = True + else: + d[key] = False + if key_type_map[key] is str: + d[key] = str(value) + if key_type_map[key] is list: + d[key] = ast.literal_eval(value) + +def load_yaml(): + with open('templates/config_overall.yaml', 'r') as file: + return yaml.safe_load(file) + +def save_yaml(data): + convert_dict_values(data, key_type_map) + with open('config_reset/config_overall_reset.yaml', 'w') as file: + yaml.dump(data, file, default_flow_style=False, sort_keys=False) + +def render_form(data, parent_key=''): + form_html = '' + for key, value in data.items(): + full_key = f"{parent_key}.{key}" if parent_key else key + if isinstance(value, dict): # 处理字典 + form_html += f"

{key}

{render_form(value, full_key)}
" + else: + form_html += f"" + form_html += f"
" + return form_html + +@app.route('/', methods=['GET', 'POST']) +def index(): + if request.method == 'POST': + data = load_yaml() + for key in request.form: + keys = key.split('.') + temp = data + for k in keys[:-1]: + temp = temp[k] + temp[keys[-1]] = request.form[key] + save_yaml(data) + return redirect('/') + + data = load_yaml() + form_html = render_form(data) + return render_template('index_overall.html', form_html=form_html) + +if __name__ == '__main__': + app.run(debug=True) + diff --git a/tools/setConfig/templates/config_overall.yaml b/tools/setConfig/templates/config_overall.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d22dc6847dd3ee0ade13373233a91e38ec1f5770 --- /dev/null +++ b/tools/setConfig/templates/config_overall.yaml @@ -0,0 +1,145 @@ +--- +############################################### +# +# Configuration file for CSST simulation +# Overall settings +# CSST-Sim Group, 2024/01/08 +# +############################################### + +# Base diretories and naming setup +# can add some of the command-line arguments here as well; +# ok to pass either way or both, as long as they are consistent +work_dir: "/public/home/fangyuedong/project/workplace/" +run_name: "ext_on" + +# Project cycle and run counter are used to name the outputs +project_cycle: 9 +run_counter: 1 + +# Run options +run_option: + # Output catalog only? + # If yes, no imaging simulation will be run. Only the catalogs + # of corresponding footprints will be generated. + out_cat_only: NO + +############################################### +# Catalog setting +############################################### +# Configure the input catalog: options should be implemented +# in the corresponding (user defined) 'Catalog' class +catalog_options: + input_path: + cat_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/" + star_cat: "starcat_C9/" + galaxy_cat: "qsocat/cat2CSSTSim_bundle-50sqDeg/" + + SED_templates_path: + star_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/starcat_C9/" + galaxy_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/sedlibs/" + AGN_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/qsocat/qsosed/" + + # Only simulate stars? + star_only: NO + + # Only simulate galaxies? + galaxy_only: NO + + # rotate galaxy ellipticity + rotateEll: 0. # [degree] + + # Whether to apply milky way extinction to galaxies + enable_mw_ext_gal: YES + planck_ebv_map: "/public/home/fangyuedong/project/ext_maps/planck/HFI_CompMap_ThermalDustModel_2048_R1.20.fits" + +############################################### +# Observation setting +############################################### +obs_setting: + # (Optional) a file of point list + # if you just want to run default pointing: + # - pointing_dir: null + # - pointing_file: null + pointing_file: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/pointing50_C9/pointing_50_1_n.dat" + + obs_config_file: "/public/home/fangyuedong/project/csst_msc_sim/config/obs_config_SCI.yaml" + + # Run specific pointing(s): + # - give a list of indexes of pointings: [ip_1, ip_2...] + # - run all pointings: null + # Note: only valid when a pointing list is specified + run_pointings: [0, 1, 2, 3, 4] + + # Whether to enable astrometric modeling + enable_astrometric_model: YES + + # Cut by saturation magnitude in which band? + cut_in_band: "z" + + # saturation magnitude margin + mag_sat_margin: -2.5 + # mag_sat_margin: -15. + + # limiting magnitude margin + mag_lim_margin: +1.0 + +############################################### +# PSF setting +############################################### +psf_setting: + + # Which PSF model to use: + # "Gauss": simple gaussian profile + # "Interp": Interpolated PSF from sampled ray-tracing data + psf_model: "Interp" + + # PSF size [arcseconds] + # radius of 80% energy encircled + # NOTE: only valid for "Gauss" PSF + # psf_rcont: 0.15 + + # path to PSF data + # NOTE: only valid for "Interp" PSF + # PSF models for photometry survey simulation + psf_pho_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/dataC6/psfCube1" + # PSF models for slitless spectrum survey simulation + psf_sls_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/SLS_PSF_PCA_fp/" + +############################################### +# Shear setting +############################################### + +shear_setting: + # Options to generate mock shear field: + # "constant": all galaxies are assigned a constant reduced shear + # "catalog": get shear values from catalog + shear_type: "constant" + + # For constant shear field + reduced_g1: 0. + reduced_g2: 0. + +############################################### +# Output options +############################################### +output_setting: + output_format: "channels" # Whether to export as 16 channels (subimages) with pre- and over-scan ("image"/"channels") + shutter_output: NO # Whether to export shutter effect 16-bit image + prnu_output: NO # Whether to export the PRNU (pixel-to-pixel flat-fielding) files + +############################################### +# Random seeds +############################################### +random_seeds: + seed_poisson: 20210601 # Seed for Poisson noise + seed_CR: 20210317 # Seed for generating random cosmic ray maps + seed_flat: 20210101 # Seed for generating random flat fields + seed_prnu: 20210102 # Seed for photo-response non-uniformity + seed_gainNonUniform: 20210202 # Seed for gain nonuniformity + seed_biasNonUniform: 20210203 # Seed for bias nonuniformity + seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity + seed_badcolumns: 20240309 # Seed for bad columns + seed_defective: 20210304 # Seed for defective (bad) pixels + seed_readout: 20210601 # Seed for read-out gaussian noise +... \ No newline at end of file diff --git a/tools/setConfig/templates/index_obs.html b/tools/setConfig/templates/index_obs.html new file mode 100644 index 0000000000000000000000000000000000000000..5199558d997fceadbd7caa31daa7f39b19508254 --- /dev/null +++ b/tools/setConfig/templates/index_obs.html @@ -0,0 +1,246 @@ + + + + + + csst_msc_sim_CONF + + + + +
+

配置obs_config_SCI.yaml

+

说明: + 1,该脚本可用于生成CSST主巡天成像仿真的参数文件,相关参数说明见本页脚注;2,用户必须修改相关路径参数，其他参数可参考默认值 +. +

+
+
+ {{ form_html | safe }} + +
+
+ + + + + diff --git a/tools/setConfig/templates/index_overall.html b/tools/setConfig/templates/index_overall.html new file mode 100644 index 0000000000000000000000000000000000000000..d61ccf82cd4be5e981c9b57a09eb19bc9d64605d --- /dev/null +++ b/tools/setConfig/templates/index_overall.html @@ -0,0 +1,258 @@ + + + + + + csst_msc_sim_CONF + + + + +
+

配置config_overall.yaml

+

说明: + 1,该脚本可用于生成CSST主巡天成像仿真的参数文件,相关参数说明见本页脚注;2,用户必须修改相关路径参数，其他参数可参考默认值. +

+
+
+ {{ form_html | safe }} + +
+
+ + + + + diff --git a/tools/setConfig/templates/obs_config_SCI.yaml b/tools/setConfig/templates/obs_config_SCI.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef82bcdc6c0e17cc788ca396dd4fa3be087a159a --- /dev/null +++ b/tools/setConfig/templates/obs_config_SCI.yaml @@ -0,0 +1,85 @@ +--- +############################################### +# +# Configuration file for CSST simulation +# For single exposure type: +# SCI-WIDE +# CSST-Sim Group, 2024/01/08 +# +############################################### + +# Observation type +obs_type: "SCI" +obs_type_code: "101" +obs_id: "00000001" # this setting will only be used if pointing list file is not given + +# Define list of chips +# run_chips: [6,7,8,9,11,12,13,14,15,16,17,18,19,20,22,23,24,25] # Photometric chips +#run_chips: [1,2,3,4,5,10,21,26,27,28,29,30] # Spectroscopic chips +run_chips: [17, 22] + +# Define observation sequence +call_sequence: + # Accumulate fluxes from objects + scie_obs: + # [Optional]: exposure time of the pointing will be used as default. + # Set it here is you want to override the default + # exptime: 150. # [s] + shutter_effect: YES + flat_fielding: YES + field_dist: YES + # Accumulate fluxes from sky background + sky_background: + # [Optional]: exposure time of the pointing will be used as default. + # Set it here is you want to override the default + # exptime: 150. # [s] + shutter_effect: YES + flat_fielding: YES + enable_straylight_model: YES + # flat_level: set the total skybackground value (e-) in the exptime,if none,set null, or delete the key + # flat_level_filt: the vale of "flat_level" is in the filter "flat_level_filt", can set NUV, u, g, r, i, z, y, if + # none,set null,or delete the key + flat_level: null + flat_level_filt: null + # Apply PRNU to accumulated photons + PRNU_effect: {} + # Accumulate photons caused by cosmic rays + cosmic_rays: + # [Optional]: exposure time of the pointing will be used as default. + # Set it here is you want to override the default + # exptime: 150. # [s] + save_cosmic_img: YES # # Whether to export cosmic ray image + # Add Poission noise and dark current + poisson_and_dark: + # [Optional]: exposure time of the pointing will be used as default. + # Set it here is you want to override the default + # exptime: 150. # [s] + add_dark: YES + # Simulate brighter fatter effects + bright_fatter: {} + # Add detector defects: hot/warm pixels, bad columns + detector_defects: + hot_pixels: YES + dead_pixels: YES + bad_columns: YES + # Apply response nonlinearity + nonlinearity: {} + # Apply CCD Saturation & Blooming + blooming: {} + # Run CTE simulation + # CTE_effect: {} + # Add prescan and overscan + prescan_overscan: + add_dark: YES + # Add bias + bias: + bias_16channel: YES + # Add readout noise + readout_noise: {} + # Apply gain + gain: + gain_16channel: YES + # Output the final image + quantization_and_output: + format_output: YES +...