From 26a0eb7de04da47cb9587f9f39ac3c4dc6307336 Mon Sep 17 00:00:00 2001 From: Chengliang Date: Thu, 14 Nov 2024 18:32:24 +0800 Subject: [PATCH] reset config.yaml with html --- tools/setConfig/.DS_Store | Bin 0 -> 8196 bytes tools/setConfig/reset_obs.py | 118 ++++++++ tools/setConfig/reset_overall.py | 127 +++++++++ tools/setConfig/templates/config_overall.yaml | 145 ++++++++++ tools/setConfig/templates/index_obs.html | 246 +++++++++++++++++ tools/setConfig/templates/index_overall.html | 258 ++++++++++++++++++ tools/setConfig/templates/obs_config_SCI.yaml | 85 ++++++ 7 files changed, 979 insertions(+) create mode 100644 tools/setConfig/.DS_Store create mode 100644 tools/setConfig/reset_obs.py create mode 100644 tools/setConfig/reset_overall.py create mode 100644 tools/setConfig/templates/config_overall.yaml create mode 100644 tools/setConfig/templates/index_obs.html create mode 100644 tools/setConfig/templates/index_overall.html create mode 100644 tools/setConfig/templates/obs_config_SCI.yaml diff --git a/tools/setConfig/.DS_Store b/tools/setConfig/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..316c88df0f1de73f1e37a841b6e985fdc1b95458 GIT binary patch literal 8196 zcmeHMzi-n(6n@u+)FDL#l}hb^WC_Nqv;|clmXHRCt{A}pG{j9vA{?KU9}QI$lno?y zHY6q{Bqk*i#k*0^qEyi20T$fhr1r%cRlXTXqyv;0mFb{z%XDKFbo_A2Jp-l#Y%bZtE;9p z3>XIfO9sULV516I)N-t(+&a+66aX@V)3TtCJV0V>EsI)?m6TA}RM`X5P^ME1rr@}4 z3w6k%mSZIqoS1?W(<3vTp)ff*aJEz@R#eioh5^ICECV8U&r%hcdqC^4{9T07b=0?C zWs%@K?Gq=T9#M-LKpXckF-*Oy*hPK zd#M)R#a6QzvpBU7T@QN!+y1fQ_Ll z``su)vKpRW0^(l?!-Qo;1F$J*PS6U6-p##Nlh8}l33G5W*9IG{4xV9Q+CQFn3=p4#x06m+eUqeDvHn>D=8sp tWI7Hh({ae@ABO1LP!*V>mSZJx2IY@`2>4rG!xNsu=J_vO#x%cyfp4pI{cr#P literal 0 HcmV?d00001 diff --git a/tools/setConfig/reset_obs.py b/tools/setConfig/reset_obs.py new file mode 100644 index 0000000..c8c7a65 --- /dev/null +++ b/tools/setConfig/reset_obs.py @@ -0,0 +1,118 @@ +from flask import Flask, render_template, request, redirect +import yaml +import ast + +app = Flask(__name__) + + +key_type_map = { + 'obs_type': str, + 'obs_type_code': str, + 'obs_id': str, + 'run_chips': list, + 'call_sequence':{ + 'scie_obs':{ + 'shutter_effect': bool, + 'flat_fielding': bool, + 'field_dist': bool, + }, + 'sky_background':{ + 'shutter_effect': bool, + 'flat_fielding': bool, + 'enable_straylight_model': bool, + 'flat_level': None, + 'flat_level_filt': None, + }, + 'PRNU_effect': {}, + 'cosmic_rays':{ + 'save_cosmic_img': bool, + }, + 'poisson_and_dark':{ + 'add_dark': bool, + }, + 'bright_fatter': {}, + 'detector_defects':{ + 'hot_pixels': bool, + 'dead_pixels': bool, + 'bad_columns': bool, + }, + 'nonlinearity': {}, + 'blooming': {}, + 'prescan_overscan':{ + 'add_dark': bool, + }, + 'bias':{ + 'bias_16channel': bool, + }, + 'readout_noise': {}, + 'gain':{ + 'gain_16channel': bool, + }, + 'quantization_and_output':{ + 'format_output': bool, + }, + }, +} +def convert_dict_values(d, key_type_map): + for key, value in d.items(): + if isinstance(value, dict): + convert_dict_values(value, key_type_map[key]) + elif key in key_type_map: + if key_type_map[key] is int: + d[key] = int(value) + if key_type_map[key] is float: + d[key] = float(value) + if key_type_map[key] is bool: + if d[key].lower() == 'yes' or d[key].lower() == 'true': + d[key] = True + else: + d[key] = False + if key_type_map[key] is str: + d[key] = str(value) + if key_type_map[key] is list: + d[key] = ast.literal_eval(value) + if key_type_map[key] is None: + d[key] = None + + + +def load_yaml(): + with open('templates/obs_config_SCI.yaml', 'r') as file: + return yaml.safe_load(file) + +def save_yaml(data): + convert_dict_values(data, key_type_map) + with open('config_reset/obs_config_SCI_reset.yaml', 'w') as file: + yaml.dump(data, file, default_flow_style=False, sort_keys=False) + +def render_form(data, parent_key=''): + form_html = '' + for key, value in data.items(): + full_key = f"{parent_key}.{key}" if parent_key else key + if isinstance(value, dict): # 处理字典 + form_html += f"

{key}

{render_form(value, full_key)}
" + else: + form_html += f"" + form_html += f"
" + return form_html + +@app.route('/', methods=['GET', 'POST']) +def index(): + if request.method == 'POST': + data = load_yaml() + for key in request.form: + keys = key.split('.') + temp = data + for k in keys[:-1]: + temp = temp[k] + temp[keys[-1]] = request.form[key] + save_yaml(data) + return redirect('/') + + data = load_yaml() + form_html = render_form(data) + return render_template('index_obs.html', form_html=form_html) + +if __name__ == '__main__': + app.run(debug=True) + diff --git a/tools/setConfig/reset_overall.py b/tools/setConfig/reset_overall.py new file mode 100644 index 0000000..df51b24 --- /dev/null +++ b/tools/setConfig/reset_overall.py @@ -0,0 +1,127 @@ +from flask import Flask, render_template, request, redirect +import yaml +import ast + +app = Flask(__name__) + +key_type_map = { + 'work_dir': str, + 'run_name': str, + 'project_cycle': int, + 'run_counter': int, + 'run_option':{ + 'out_cat_only': bool, + }, + 'catalog_options':{ + 'input_path':{ + 'cat_dir': str, + 'star_cat': str, + 'galaxy_cat': str, + }, + 'SED_templates_path':{ + 'star_SED': str, + 'galaxy_SED': str, + 'AGN_SED': str, + }, + 'star_only': bool, + 'galaxy_only': bool, + 'rotateEll': float, + 'enable_mw_ext_gal': bool, + 'planck_ebv_map':str, + }, + 'obs_setting':{ + 'pointing_file': str, + 'obs_config_file': str, + 'run_pointings': list, + 'enable_astrometric_model': bool, + 'cut_in_band': str, + 'mag_sat_margin': float, + 'mag_lim_margin': float, + }, + 'psf_setting':{ + 'psf_model': str, + 'psf_pho_dir': str, + 'psf_sls_dir': str, + }, + 'shear_setting':{ + 'shear_type': str, + 'reduced_g1': float, + 'reduced_g2': float, + }, + 'output_setting':{ + 'output_format': str, + 'shutter_output': bool, + 'prnu_output': bool, + }, + 'random_seeds':{ + 'seed_poisson': int, + 'seed_CR': int, + 'seed_flat': int, + 'seed_prnu': int, + 'seed_gainNonUniform': int, + 'seed_biasNonUniform': int, + 'seed_rnNonUniform': int, + 'seed_badcolumns': int, + 'seed_defective': int, + 'seed_readout': int, + }, +} +def convert_dict_values(d, key_type_map): + for key, value in d.items(): + if isinstance(value, dict): + convert_dict_values(value, key_type_map[key]) + elif key in key_type_map: + if key_type_map[key] is int: + d[key] = int(value) + if key_type_map[key] is float: + d[key] = float(value) + if key_type_map[key] is bool: + if d[key].lower() == 'yes' or d[key].lower() == 'true': + d[key] = True + else: + d[key] = False + if key_type_map[key] is str: + d[key] = str(value) + if key_type_map[key] is list: + d[key] = ast.literal_eval(value) + +def load_yaml(): + with open('templates/config_overall.yaml', 'r') as file: + return yaml.safe_load(file) + +def save_yaml(data): + convert_dict_values(data, key_type_map) + with open('config_reset/config_overall_reset.yaml', 'w') as file: + yaml.dump(data, file, default_flow_style=False, sort_keys=False) + +def render_form(data, parent_key=''): + form_html = '' + for key, value in data.items(): + full_key = f"{parent_key}.{key}" if parent_key else key + if isinstance(value, dict): # 处理字典 + form_html += f"

{key}

{render_form(value, full_key)}
" + else: + form_html += f"" + form_html += f"
" + return form_html + +@app.route('/', methods=['GET', 'POST']) +def index(): + if request.method == 'POST': + data = load_yaml() + for key in request.form: + keys = key.split('.') + temp = data + for k in keys[:-1]: + temp = temp[k] + temp[keys[-1]] = request.form[key] + save_yaml(data) + return redirect('/') + + data = load_yaml() + form_html = render_form(data) + return render_template('index_overall.html', form_html=form_html) + +if __name__ == '__main__': + app.run(debug=True) + diff --git a/tools/setConfig/templates/config_overall.yaml b/tools/setConfig/templates/config_overall.yaml new file mode 100644 index 0000000..d22dc68 --- /dev/null +++ b/tools/setConfig/templates/config_overall.yaml @@ -0,0 +1,145 @@ +--- +############################################### +# +# Configuration file for CSST simulation +# Overall settings +# CSST-Sim Group, 2024/01/08 +# +############################################### + +# Base diretories and naming setup +# can add some of the command-line arguments here as well; +# ok to pass either way or both, as long as they are consistent +work_dir: "/public/home/fangyuedong/project/workplace/" +run_name: "ext_on" + +# Project cycle and run counter are used to name the outputs +project_cycle: 9 +run_counter: 1 + +# Run options +run_option: + # Output catalog only? + # If yes, no imaging simulation will be run. Only the catalogs + # of corresponding footprints will be generated. + out_cat_only: NO + +############################################### +# Catalog setting +############################################### +# Configure the input catalog: options should be implemented +# in the corresponding (user defined) 'Catalog' class +catalog_options: + input_path: + cat_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/" + star_cat: "starcat_C9/" + galaxy_cat: "qsocat/cat2CSSTSim_bundle-50sqDeg/" + + SED_templates_path: + star_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/starcat_C9/" + galaxy_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/sedlibs/" + AGN_SED: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/qsocat/qsosed/" + + # Only simulate stars? + star_only: NO + + # Only simulate galaxies? + galaxy_only: NO + + # rotate galaxy ellipticity + rotateEll: 0. # [degree] + + # Whether to apply milky way extinction to galaxies + enable_mw_ext_gal: YES + planck_ebv_map: "/public/home/fangyuedong/project/ext_maps/planck/HFI_CompMap_ThermalDustModel_2048_R1.20.fits" + +############################################### +# Observation setting +############################################### +obs_setting: + # (Optional) a file of point list + # if you just want to run default pointing: + # - pointing_dir: null + # - pointing_file: null + pointing_file: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/pointing50_C9/pointing_50_1_n.dat" + + obs_config_file: "/public/home/fangyuedong/project/csst_msc_sim/config/obs_config_SCI.yaml" + + # Run specific pointing(s): + # - give a list of indexes of pointings: [ip_1, ip_2...] + # - run all pointings: null + # Note: only valid when a pointing list is specified + run_pointings: [0, 1, 2, 3, 4] + + # Whether to enable astrometric modeling + enable_astrometric_model: YES + + # Cut by saturation magnitude in which band? + cut_in_band: "z" + + # saturation magnitude margin + mag_sat_margin: -2.5 + # mag_sat_margin: -15. + + # limiting magnitude margin + mag_lim_margin: +1.0 + +############################################### +# PSF setting +############################################### +psf_setting: + + # Which PSF model to use: + # "Gauss": simple gaussian profile + # "Interp": Interpolated PSF from sampled ray-tracing data + psf_model: "Interp" + + # PSF size [arcseconds] + # radius of 80% energy encircled + # NOTE: only valid for "Gauss" PSF + # psf_rcont: 0.15 + + # path to PSF data + # NOTE: only valid for "Interp" PSF + # PSF models for photometry survey simulation + psf_pho_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/dataC6/psfCube1" + # PSF models for slitless spectrum survey simulation + psf_sls_dir: "/public/share/yangxuliu/CSSOSDataProductsSims/data_50sqDeg/SLS_PSF_PCA_fp/" + +############################################### +# Shear setting +############################################### + +shear_setting: + # Options to generate mock shear field: + # "constant": all galaxies are assigned a constant reduced shear + # "catalog": get shear values from catalog + shear_type: "constant" + + # For constant shear field + reduced_g1: 0. + reduced_g2: 0. + +############################################### +# Output options +############################################### +output_setting: + output_format: "channels" # Whether to export as 16 channels (subimages) with pre- and over-scan ("image"/"channels") + shutter_output: NO # Whether to export shutter effect 16-bit image + prnu_output: NO # Whether to export the PRNU (pixel-to-pixel flat-fielding) files + +############################################### +# Random seeds +############################################### +random_seeds: + seed_poisson: 20210601 # Seed for Poisson noise + seed_CR: 20210317 # Seed for generating random cosmic ray maps + seed_flat: 20210101 # Seed for generating random flat fields + seed_prnu: 20210102 # Seed for photo-response non-uniformity + seed_gainNonUniform: 20210202 # Seed for gain nonuniformity + seed_biasNonUniform: 20210203 # Seed for bias nonuniformity + seed_rnNonUniform: 20210204 # Seed for readout-noise nonuniformity + seed_badcolumns: 20240309 # Seed for bad columns + seed_defective: 20210304 # Seed for defective (bad) pixels + seed_readout: 20210601 # Seed for read-out gaussian noise +... \ No newline at end of file diff --git a/tools/setConfig/templates/index_obs.html b/tools/setConfig/templates/index_obs.html new file mode 100644 index 0000000..5199558 --- /dev/null +++ b/tools/setConfig/templates/index_obs.html @@ -0,0 +1,246 @@ + + + + + + csst_msc_sim_CONF + + + + +
+

配置obs_config_SCI.yaml

+

说明: + 1,该脚本可用于生成CSST主巡天成像仿真的参数文件,相关参数说明见本页脚注;2,用户必须修改相关路径参数，其他参数可参考默认值 +. +

+
+
+ {{ form_html | safe }} + +
+
+ +
+

这是一个有用的网页

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KEYSVALUESCOMMENTS
obs_type"SCI", [str]仿真图像类型
obs_type_code"101", [str]数据系统内部标号
obs_id"00000001", [str]this setting will only be used if pointing list file is not given
run_chips[7,8,9], [list]Define list of chips
scie_obs:shutter_effectYES, [bool]是否应用快门效应
scie_obs:flat_fieldingYES, [bool]是否应用平场模块
scie_obs:field_distYES, [bool]
sky_background:shutter_effectYES, [bool]是否应用快门效应
sky_background:flat_fieldingYES, [bool]是否应用平场模块
sky_background:enable_straylight_modelYES, [bool]是否应用杂散光模块
sky_background:flat_levelnull [null or int]set the total skybackground value (e-) in the exptime,if none,set null, or delete the key
sky_background:flat_level_filtnull, [str]the vale of "flat_level" is in the filter "flat_level_filt", can set NUV, u, g, r, i, z, y, if none,set null,or delete the key
PRNU_effect---应用像素间不均匀响应模块；如果需关闭，请在生成的yaml文件中直接注释
cosmic_rays:save_cosmic_imgYES, [bool]是否保存宇宙线图像
poisson_and_dark:add_darkYES, [bool]是否添加暗电流
bright_fatterYES, [bool]应用亮胖效应模块；如果需关闭，请在生成的yaml文件中直接注释
detector_defects:hot_pixelsYES, [bool]是否添加热像元
detector_defects:dead_pixelsYES, [bool]是否添加坏像元
detector_defects:bad_columnsYES, [bool]是否添加坏像列
nonlinearity---应用非线性响应模块；如果需关闭，请在生成的yaml文件中直接注释
blooming---是否应用饱和溢出模块;如果需关闭，请在生成的yaml文件中直接注释
prescan_overscan:add_darkYES, [bool] 是否在pre/over-scan区域添加暗电流/
bias:bias_16channelYES, [bool]是否添加16通道的偏置电压
readout_noise---添加读出噪声；如果需关闭，请在生成的yaml文件中直接注释
gain:gain_16channelYES, [bool]是否应用增益
quantization_and_output:format_outputYES, [bool]是否按0级数据格式定义输出图像
+
+ + + diff --git a/tools/setConfig/templates/index_overall.html b/tools/setConfig/templates/index_overall.html new file mode 100644 index 0000000..d61ccf8 --- /dev/null +++ b/tools/setConfig/templates/index_overall.html @@ -0,0 +1,258 @@ + + + + + + csst_msc_sim_CONF + + + + +
+

配置config_overall.yaml

+

说明: + 1,该脚本可用于生成CSST主巡天成像仿真的参数文件,相关参数说明见本页脚注;2,用户必须修改相关路径参数，其他参数可参考默认值. +

+
+
+ {{ form_html | safe }} + +
+
+ + + + + diff --git a/tools/setConfig/templates/obs_config_SCI.yaml b/tools/setConfig/templates/obs_config_SCI.yaml new file mode 100644 index 0000000..ef82bcd --- /dev/null +++ b/tools/setConfig/templates/obs_config_SCI.yaml @@ -0,0 +1,85 @@ +--- +############################################### +# +# Configuration file for CSST simulation +# For single exposure type: +# SCI-WIDE +# CSST-Sim Group, 2024/01/08 +# +############################################### + +# Observation type +obs_type: "SCI" +obs_type_code: "101" +obs_id: "00000001" # this setting will only be used if pointing list file is not given + +# Define list of chips +# run_chips: [6,7,8,9,11,12,13,14,15,16,17,18,19,20,22,23,24,25] # Photometric chips +#run_chips: [1,2,3,4,5,10,21,26,27,28,29,30] # Spectroscopic chips +run_chips: [17, 22] + +# Define observation sequence +call_sequence: + # Accumulate fluxes from objects + scie_obs: + # [Optional]: exposure time of the pointing will be used as default. + # Set it here is you want to override the default + # exptime: 150. # [s] + shutter_effect: YES + flat_fielding: YES + field_dist: YES + # Accumulate fluxes from sky background + sky_background: + # [Optional]: exposure time of the pointing will be used as default. + # Set it here is you want to override the default + # exptime: 150. # [s] + shutter_effect: YES + flat_fielding: YES + enable_straylight_model: YES + # flat_level: set the total skybackground value (e-) in the exptime,if none,set null, or delete the key + # flat_level_filt: the vale of "flat_level" is in the filter "flat_level_filt", can set NUV, u, g, r, i, z, y, if + # none,set null,or delete the key + flat_level: null + flat_level_filt: null + # Apply PRNU to accumulated photons + PRNU_effect: {} + # Accumulate photons caused by cosmic rays + cosmic_rays: + # [Optional]: exposure time of the pointing will be used as default. + # Set it here is you want to override the default + # exptime: 150. # [s] + save_cosmic_img: YES # # Whether to export cosmic ray image + # Add Poission noise and dark current + poisson_and_dark: + # [Optional]: exposure time of the pointing will be used as default. + # Set it here is you want to override the default + # exptime: 150. # [s] + add_dark: YES + # Simulate brighter fatter effects + bright_fatter: {} + # Add detector defects: hot/warm pixels, bad columns + detector_defects: + hot_pixels: YES + dead_pixels: YES + bad_columns: YES + # Apply response nonlinearity + nonlinearity: {} + # Apply CCD Saturation & Blooming + blooming: {} + # Run CTE simulation + # CTE_effect: {} + # Add prescan and overscan + prescan_overscan: + add_dark: YES + # Add bias + bias: + bias_16channel: YES + # Add readout noise + readout_noise: {} + # Apply gain + gain: + gain_16channel: YES + # Output the final image + quantization_and_output: + format_output: YES +... -- GitLab