Commit 6e91db29 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

refactor: 统一所有DAG配置的参数格式，更新配置加载器与测试用例

parent 3510ab18

csst_dag/v2/config_loader.py

+64 −12
Original line number Diff line number Diff line
@@ -51,27 +51,50 @@ def _pick_first_option_value(raw_value) -> str | None: 
    return option_values[0] if option_values else None





def _normalize_field_group(field_group: dict | None) -> tuple[dict, dict, dict]:

    """将字段中心的 submission 配置展开为 defaults/fixed/options 三张表。"""

_MISSING = object()





def _extract_field_default(field_cfg: dict) -> object:

    if "default" in field_cfg and field_cfg.get("default") is not None:

        return field_cfg.get("default")

    if "value" in field_cfg and field_cfg.get("value") is not None:

        return field_cfg.get("value")

    return _MISSING





def _normalize_field_group(field_group: dict | None) -> tuple[dict, dict, dict, dict, dict, dict]:

    """将字段中心的 submission 配置展开为 defaults/fixed/options/allow_empty/option_sources/default_strategies 六张表。"""

    defaults: dict = {}

    fixed: dict = {}

    options: dict = {}

    allow_empty: dict = {}

    option_sources: dict = {}

    default_strategies: dict = {}

    if not isinstance(field_group, dict):

        return defaults, fixed, options

        return defaults, fixed, options, allow_empty, option_sources, default_strategies



    for field_name, field_cfg in field_group.items():

        if isinstance(field_cfg, dict):

            if "value" in field_cfg and field_cfg.get("value") is not None:

                defaults[field_name] = field_cfg.get("value")

            if bool(field_cfg.get("fixed")) and "value" in field_cfg:

                fixed[field_name] = field_cfg.get("value")

            field_default = _extract_field_default(field_cfg)

            if field_default is not _MISSING:

                defaults[field_name] = field_default

            if bool(field_cfg.get("fixed")) and field_default is not _MISSING:

                fixed[field_name] = field_default

            option_values = _normalize_option_values(field_cfg.get("options"))

            if option_values:

                options[field_name] = option_values

            if "allow_empty" in field_cfg:

                allow_empty[field_name] = bool(field_cfg.get("allow_empty"))

            option_source = str(field_cfg.get("options_source") or "").strip()

            if option_source:

                option_sources[field_name] = option_source

            default_strategy = str(field_cfg.get("default_strategy") or "").strip()

            if default_strategy:

                default_strategies[field_name] = default_strategy

            continue

        if field_cfg is not None:

            defaults[field_name] = field_cfg

    return defaults, fixed, options

    return defaults, fixed, options, allow_empty, option_sources, default_strategies





def _normalize_match_options(match_cfg: dict | None) -> dict:
@@ -151,23 +174,52 @@ def _normalize_submission_section(submission_cfg: dict | None) -> dict: 
        "data_defaults": {},

        "data_fixed": {},

        "data_options": {},

        "data_allow_empty": {},

        "data_option_sources": {},

        "data_default_strategies": {},

        "proc_defaults": {},

        "proc_fixed": {},

        "proc_options": {},

        "proc_allow_empty": {},

        "proc_option_sources": {},

        "proc_default_strategies": {},

    }

    if not isinstance(submission_cfg, dict):

        return normalized



    # 新结构：submission.data.<field>.value/fixed 与 submission.proc.<field>.value/fixed

    data_defaults, data_fixed, data_options = _normalize_field_group(submission_cfg.get("data"))

    proc_defaults, proc_fixed, proc_options = _normalize_field_group(submission_cfg.get("proc"))

    if data_defaults or data_fixed or data_options or proc_defaults or proc_fixed or proc_options:

    # 新结构：submission.data.<field>.default/fixed 与 submission.proc.<field>.default/fixed

    data_defaults, data_fixed, data_options, data_allow_empty, data_option_sources, data_default_strategies = _normalize_field_group(

        submission_cfg.get("data")

    )

    proc_defaults, proc_fixed, proc_options, proc_allow_empty, proc_option_sources, proc_default_strategies = _normalize_field_group(

        submission_cfg.get("proc")

    )

    if (

        data_defaults

        or data_fixed

        or data_options

        or data_allow_empty

        or data_option_sources

        or data_default_strategies

        or proc_defaults

        or proc_fixed

        or proc_options

        or proc_allow_empty

        or proc_option_sources

        or proc_default_strategies

    ):

        normalized["data_defaults"] = data_defaults

        normalized["data_fixed"] = data_fixed

        normalized["data_options"] = data_options

        normalized["data_allow_empty"] = data_allow_empty

        normalized["data_option_sources"] = data_option_sources

        normalized["data_default_strategies"] = data_default_strategies

        normalized["proc_defaults"] = proc_defaults

        normalized["proc_fixed"] = proc_fixed

        normalized["proc_options"] = proc_options

        normalized["proc_allow_empty"] = proc_allow_empty

        normalized["proc_option_sources"] = proc_option_sources

        normalized["proc_default_strategies"] = proc_default_strategies

        return normalized



    # 兼容旧结构：data_defaults/data_fixed/proc_defaults/proc_fixed

csst_dag/v2/dag_config/csst-cpic-l1-qc0.yml

+78 −22
Original line number Diff line number Diff line
@@ -20,43 +20,99 @@ default_params_input: '{"dag":"csst-cpic-l1-qc0","dag_run":"csst-cpic-l1-qc0-int 
submission:

  data:

    dataset:

      value: "test-dataset"

      default: test-dataset

      options: []

      fixed: false

      allow_empty: false

    source_batch_id:

      value: "default"

      default: default

      options: []

      fixed: false

      allow_empty: false

    data_model:

      value: "raw"

      default: raw

      options: []

      fixed: true

      allow_empty: false

    instrument:

      value: "CPIC"

      default: CPIC

      options:

      - CPIC

      fixed: true

      allow_empty: false

    obs_type:

      value: "SCI"

      default: SCI

      options:

      - SCI

      - DSF

      - CALS

      - BIAS

      - DARK

      - FLAT

      - BKG

      - LASER

      fixed: false

      allow_empty: false

    obs_group:

      value: "G1"

      default: G1

      options: []

      fixed: false

      allow_empty: true

    obs_id:

      value: "123456"

      default: '123456'

      options: []

      fixed: false

      allow_empty: true

    detector:

      value: "VIS"

      default: VIS

      options:

      - VIS

      - NIR

      fixed: false

      allow_empty: true

    filter:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    healpix:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    custom_id:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    qc_status:

      value: ""

      default: ''

      options: []

      fixed: false

      allow_empty: true

    prc_status:

      value: ""

      default: ''

      options: []

      fixed: false

      allow_empty: true

  proc:

    pmapname:

      value: "csst_000155.pmap"

      options: []

      options_source: ccds_pmaps

      default_strategy: source_default

      fixed: false

      allow_empty: true

    ref_cat:

      value: "trilegal_093"

      options: []

      options_source: dfs_catalog_names

      default_strategy: first

      fixed: false

      allow_empty: true

    extra_kwargs:

      value: "{}"



      default: '{}'

      options: []

      fixed: false

      allow_empty: true

dag_timeout_hours: 15

# 容器任务定义（按 dependencies 串联）

tasks:

csst_dag/v2/dag_config/csst-cpic-l1.yml

+78 −22
Original line number Diff line number Diff line
@@ -21,43 +21,99 @@ default_params_input: '{"dag":"csst-cpic-l1","dag_run":"csst-cpic-l1-inttest","d 
submission:

  data:

    dataset:

      value: "test-dataset"

      default: test-dataset

      options: []

      fixed: false

      allow_empty: false

    source_batch_id:

      value: "default"

      default: default

      options: []

      fixed: false

      allow_empty: false

    data_model:

      value: "raw"

      default: raw

      options: []

      fixed: true

      allow_empty: false

    instrument:

      value: "CPIC"

      default: CPIC

      options:

      - CPIC

      fixed: true

      allow_empty: false

    obs_type:

      value: "SCI"

      default: SCI

      options:

      - SCI

      - DSF

      - CALS

      - BIAS

      - DARK

      - FLAT

      - BKG

      - LASER

      fixed: false

      allow_empty: false

    obs_group:

      value: "G1"

      default: G1

      options: []

      fixed: false

      allow_empty: true

    obs_id:

      value: "123456"

      default: '123456'

      options: []

      fixed: false

      allow_empty: true

    detector:

      value: "VIS"

      default: VIS

      options:

      - VIS

      - NIR

      fixed: false

      allow_empty: true

    filter:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    healpix:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    custom_id:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    qc_status:

      value: ""

      default: ''

      options: []

      fixed: false

      allow_empty: true

    prc_status:

      value: ""

      default: ''

      options: []

      fixed: false

      allow_empty: true

  proc:

    pmapname:

      value: "csst_000155.pmap"

      options: []

      options_source: ccds_pmaps

      default_strategy: source_default

      fixed: false

      allow_empty: true

    ref_cat:

      value: "trilegal_093"

      options: []

      options_source: dfs_catalog_names

      default_strategy: first

      fixed: false

      allow_empty: true

    extra_kwargs:

      value: "{}"



      default: '{}'

      options: []

      fixed: false

      allow_empty: true

dag_timeout_hours: 15

# 容器任务定义（按 dependencies 串联）

tasks:

csst_dag/v2/dag_config/csst-echo.yml

+107 −63
Original line number Diff line number Diff line
@@ -20,87 +20,131 @@ default_params_input: '{"dag":"csst-echo","dag_run":"csst-echo-inttest","dataset 
submission:

  data:

    dataset:

      value: "test-msc-c9-25sqdeg-v3"

      default: test-msc-c9-25sqdeg-v3

      options: []

      fixed: false

      allow_empty: false

    source_batch_id:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: false

    data_model:

      value: "raw"

      default: raw

      options: []

      fixed: true

      allow_empty: false

    instrument:

      value: "MSC"

      default: MSC

      options:

      - MSC

      fixed: true

      allow_empty: false

    obs_type:

      value: "WIDE"

      default: WIDE

      options:

        - "WIDE"

        - "DEEP"

      - WIDE

      - DEEP

      fixed: false

      allow_empty: false

    obs_group:

      value: "W5"

      default: W5

      options: []

      fixed: false

      allow_empty: true

    obs_id:

      value: "10100131914"

      default: '10100131914'

      options: []

      fixed: false

      allow_empty: true

    detector:

      value: "09"

      default: '09'

      options:

        - "01"

        - "02"

        - "03"

        - "04"

        - "05"

        - "06"

        - "07"

        - "08"

        - "09"

        - "10"

        - "11"

        - "12"

        - "13"

        - "14"

        - "15"

        - "16"

        - "17"

        - "18"

        - "19"

        - "20"

        - "21"

        - "22"

        - "23"

        - "24"

        - "25"

        - "26"

        - "27"

        - "28"

        - "29"

        - "30"

      - '01'

      - '02'

      - '03'

      - '04'

      - '05'

      - '06'

      - '07'

      - '08'

      - '09'

      - '10'

      - '11'

      - '12'

      - '13'

      - '14'

      - '15'

      - '16'

      - '17'

      - '18'

      - '19'

      - '20'

      - '21'

      - '22'

      - '23'

      - '24'

      - '25'

      - '26'

      - '27'

      - '28'

      - '29'

      - '30'

      fixed: false

      allow_empty: true

    filter:

      default: ''

      options:

        - "NUV"

        - "GU"

        - "GV"

        - "GI"

        - "u"

        - "g"

        - "r"

        - "i"

        - "z"

        - "y"

      - NUV

      - GU

      - GV

      - GI

      - u

      - g

      - r

      - i

      - z

      - y

      fixed: false

      allow_empty: true

    healpix:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    custom_id:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    qc_status:

      value: ""

      default: ''

      options: []

      fixed: false

      allow_empty: true

    prc_status:

      value: ""

      default: ''

      options: []

      fixed: false

      allow_empty: true

  proc:

    pmapname:

      value: "csst_000155.pmap"

      options: []

      options_source: ccds_pmaps

      default_strategy: source_default

      fixed: false

      allow_empty: true

    ref_cat:

      value: "trilegal_093"

      options: []

      options_source: dfs_catalog_names

      default_strategy: first

      fixed: false

      allow_empty: true

    extra_kwargs:

      value: "{}"



      default: '{}'

      options: []

      fixed: false

      allow_empty: true

dag_timeout_hours: 15

# 容器任务定义

tasks:

csst_dag/v2/dag_config/csst-hstdm-l1.yml

+76 −32
Original line number Diff line number Diff line
@@ -21,49 +21,93 @@ default_params_input: '{"dag":"csst-hstdm-l1","dag_run":"csst-hstdm-l1-inttest", 
submission:

  data:

    dataset:

      value: "test-dataset"

      default: test-dataset

      options: []

      fixed: false

      allow_empty: false

    source_batch_id:

      value: "default"

      default: default

      options: []

      fixed: false

      allow_empty: false

    data_model:

      value: "raw"

      default: raw

      options: []

      fixed: true

      allow_empty: false

    instrument:

      value: "HSTDM"

      default: HSTDM

      options:

      - HSTDM

      fixed: true

      allow_empty: false

    obs_type:

      value: "STARE"

      default: STARE

      options:

        - "STARE"

        - "OTF"

      - STARE

      - OTF

      fixed: false

      allow_empty: false

    obs_group:

      value: "G1"

      default: G1

      options: []

      fixed: false

      allow_empty: true

    obs_id:

      value: "123456"

      default: '123456'

      options: []

      fixed: false

      allow_empty: true

    detector:

      value: "SIS1"

      default: SIS1

      options:

        - "SIS1"

        - "SIS2"

      - SIS1

      - SIS2

      fixed: false

      allow_empty: true

    filter:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    healpix:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    custom_id:

      value: ""

      default: ''

      options: []

      fixed: true

      allow_empty: true

    qc_status:

      value: ""

      default: ''

      options: []

      fixed: false

      allow_empty: true

    prc_status:

      value: ""

      default: ''

      options: []

      fixed: false

      allow_empty: true

  proc:

    pmapname:

      value: "csst_000155.pmap"

      options: []

      options_source: ccds_pmaps

      default_strategy: source_default

      fixed: false

      allow_empty: true

    ref_cat:

      value: "trilegal_093"

      options: []

      options_source: dfs_catalog_names

      default_strategy: first

      fixed: false

      allow_empty: true

    extra_kwargs:

      value: "{}"



      default: '{}'

      options: []

      fixed: false

      allow_empty: true

dag_timeout_hours: 15

# 容器任务定义（按 dependencies 串联）

tasks: