Commit e2b2dbf1 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

fix(deploy): 更新环境配置并修复TOML解析回退逻辑

parent 159008a7
Loading
Loading
Loading
Loading
+9 −3
Original line number Diff line number Diff line
@@ -80,12 +80,17 @@ make ansible-down
说明:
- `make pull` 只负责从 `required-images.txt` 拉取第三方镜像,不依赖 `IMAGE_TAG/BUILD_NUMBER/HARBOR_PROJECT`
- `make push` 只负责把已有 tag 的镜像推送到仓库,不依赖 `BUILD_NUMBER`
- `make ansible-*` 主要依赖 `ENV` 选择 inventory;镜像来源与 tag 由 `deploy_configs/<env>/inventory.ini` 中的 `harbor_project/image_tag` 控制
- `make ansible-*` 主要依赖 `ENV` 选择 inventory;同时会把 `IMAGE_TAG`(必传,默认 `latest`)与 `HARBOR_PROJECT`(可选,镜像仓库前缀,如 `harbor.csst.nao:10443/csst/`)作为 Ansible extra vars 传入,用于决定 docker compose 使用的镜像 tag/前缀

底层执行的命令形式为:

```bash
ansible-playbook -i deploy_configs/<env>/inventory.ini ansible/<playbook>.yml -e env=<env>
ansible-playbook \
  -i deploy_configs/<env>/inventory.ini \
  ansible/<playbook>.yml \
  -e env=<env> \
  -e image_tag=<image_tag> \
  [-e harbor_project=<harbor_project_prefix>]
```

### 私有 Harbor 镜像拉取(Docker login)
@@ -123,7 +128,8 @@ ansible-playbook -i deploy_configs/${ENV}/inventory.ini ansible/update.yml -e en
- 存放 Airflow 容器基础静态配置(如 `AIRFLOW_UID`)。

2) `deploy_configs/<env>/inventory.ini`(必需)
- 定义部署拓扑(master/worker)、SSH 连接与关键变量(如 `deploy_dir``harbor_project``image_tag``jwt_secret`)。`worker` 组支持按主机单独设置 `worker_concurrency`,Ansible 在启动各 worker 时会把该值映射到容器内并发配置。
- 定义部署拓扑(master/worker)、SSH 连接与关键变量(如 `deploy_dir``registry_pull``harbor_registry``extra_hosts``jwt_secret`)。`worker` 组支持按主机单独设置 `worker_concurrency`,Ansible 在启动各 worker 时会把该值映射到容器内并发配置。
- 镜像 tag/仓库前缀默认由执行端的 `IMAGE_TAG/HARBOR_PROJECT` 传入(见上文);如需固化到环境配置,也可以在 inventory 的 `[all:vars]` 中设置 `image_tag/harbor_project` 覆盖。

3) `deploy_configs/<env>/variables.toml`(可选,但推荐)
- `runtime`:唯一的共享运行时环境分组。部署时会同时导入 Airflow Variables(键名固定为 `runtime`)并渲染为 `deploy_configs/<env>/gateway.env`,分别供 DAG 任务执行阶段与 API Gateway 启动阶段使用。
+6 −0
Original line number Diff line number Diff line
@@ -12,5 +12,11 @@ csu-worker1 ansible_host=192.168.25.14 ansible_user=root deploy_dir=/opt/csst-ai
# The destination directory on remote nodes where the project will be deployed
deploy_dir=/opt/csst-airflow

registry_pull=false

harbor_registry=harbor.csst.nao:10443

extra_hosts=harbor.csst.nao:10.80.1.21

# API Gateway 与 Airflow Webserver 通信所用的 JWT 密钥 (各环境可独立生成以保证安全)
jwt_secret=eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ
+6 −5
Original line number Diff line number Diff line
@@ -5,15 +5,16 @@ CSST_DFS_ROOT = "/nfs/dfs"

CCDS_SERVER_URL = "http://192.168.25.89:29000"
CCDS_ROOT = "/nfs/ccds"
CCDS_USER=USER
CCDS_PASS=PASS
CCDS_USER = "USER"
CCDS_PASS = "PASS"

HARBOR_PROJECT = "harbor.csst.nao:10443/csst"
HARBOR_PROJECT = "csu-harbor.csst.nao:10443/csst"

HARBOR_API_URL = "https://harbor.csst.nao:10443"
HARBOR_API_URL = "https://csu-harbor.csst.nao:10443"
HARBOR_API_PROJECT = "csst"
HARBOR_API_VERIFY_SSL = false

DUMPDATA=true
DUMPDATA=false
VERBOSE=true

CSST_PORTAL_API_GATEWAY = "192.168.25.90:38501"
+7 −0
Original line number Diff line number Diff line
@@ -5,6 +5,8 @@ CSST_DFS_ROOT = "/nfs/dfs"

CCDS_SERVER_URL = "http://10.73.0.27:29000"
CCDS_ROOT = "/nfs/ccds"
CCDS_USER="USER"
CCDS_PASS="PASS"

HARBOR_PROJECT = "harbor.csst.nao:10443/csst"

@@ -12,6 +14,11 @@ HARBOR_API_URL = "https://harbor.csst.nao:10443"
HARBOR_API_PROJECT = "csst"
HARBOR_API_VERIFY_SSL = false

DUMPDATA=false
VERBOSE=true

CSST_PORTAL_API_GATEWAY = "192.168.25.90:38501"

[dag]
default_retry_delay = 300
max_active_runs = 5
+111 −3
Original line number Diff line number Diff line
@@ -6,18 +6,126 @@ from __future__ import annotations
import argparse
import json
import sys
import tomllib
from pathlib import Path
from typing import Any


def _strip_inline_comment(line: str) -> str:
    in_single = False
    in_double = False
    escaped = False
    out: list[str] = []
    for ch in line:
        if escaped:
            out.append(ch)
            escaped = False
            continue
        if ch == "\\":
            out.append(ch)
            escaped = True
            continue
        if ch == "'" and not in_double:
            in_single = not in_single
            out.append(ch)
            continue
        if ch == '"' and not in_single:
            in_double = not in_double
            out.append(ch)
            continue
        if ch == "#" and not in_single and not in_double:
            break
        out.append(ch)
    return "".join(out).strip()


def _unescape_basic_string(value: str) -> str:
    value = value.replace("\\\\", "\\")
    value = value.replace('\\"', '"')
    value = value.replace("\\n", "\n")
    value = value.replace("\\t", "\t")
    value = value.replace("\\r", "\r")
    return value


def _parse_simple_value(raw: str) -> Any:
    raw = raw.strip()
    if not raw:
        return ""

    if raw.startswith('"') and raw.endswith('"') and len(raw) >= 2:
        return _unescape_basic_string(raw[1:-1])
    if raw.startswith("'") and raw.endswith("'") and len(raw) >= 2:
        return raw[1:-1]

    lowered = raw.lower()
    if lowered == "true":
        return True
    if lowered == "false":
        return False
    if lowered == "null":
        return None

    try:
        return int(raw)
    except Exception:
        pass

    try:
        return float(raw)
    except Exception:
        pass

    return raw


def _load_toml_fallback(file_path: Path) -> dict[str, Any]:
    current_section: dict[str, Any] | None = None
    data: dict[str, Any] = {}

    for raw_line in file_path.read_text(encoding="utf-8").splitlines():
        line = _strip_inline_comment(raw_line).strip()
        if not line:
            continue
        if line.startswith("[") and line.endswith("]"):
            section_name = line[1:-1].strip()
            if not section_name:
                continue
            section = data.get(section_name)
            if not isinstance(section, dict):
                section = {}
                data[section_name] = section
            current_section = section
            continue

        if "=" not in line:
            continue

        key_raw, value_raw = line.split("=", 1)
        key = key_raw.strip()
        if not key:
            continue
        value = _parse_simple_value(value_raw)

        if current_section is None:
            data[key] = value
        else:
            current_section[key] = value

    return data


def load_variables_toml(file_path: Path) -> dict[str, Any]:
    """读取 TOML;文件不存在时返回空配置,便于部署流程兜底。"""
    if not file_path.exists():
        return {}

    try:
        import tomllib  # type: ignore

        with file_path.open("rb") as file:
            data = tomllib.load(file)
    except ModuleNotFoundError:
        data = _load_toml_fallback(file_path)

    if not isinstance(data, dict):
        raise ValueError(f"variables.toml 顶层结构必须是 table: {file_path}")