Loading README.md +9 −3 Original line number Diff line number Diff line Loading @@ -80,12 +80,17 @@ make ansible-down 说明: - `make pull` 只负责从 `required-images.txt` 拉取第三方镜像,不依赖 `IMAGE_TAG/BUILD_NUMBER/HARBOR_PROJECT`。 - `make push` 只负责把已有 tag 的镜像推送到仓库,不依赖 `BUILD_NUMBER`。 - `make ansible-*` 主要依赖 `ENV` 选择 inventory;镜像来源与 tag 由 `deploy_configs/<env>/inventory.ini` 中的 `harbor_project/image_tag` 控制。 - `make ansible-*` 主要依赖 `ENV` 选择 inventory;同时会把 `IMAGE_TAG`(必传,默认 `latest`)与 `HARBOR_PROJECT`(可选,镜像仓库前缀,如 `harbor.csst.nao:10443/csst/`)作为 Ansible extra vars 传入,用于决定 docker compose 使用的镜像 tag/前缀。 底层执行的命令形式为: ```bash ansible-playbook -i deploy_configs/<env>/inventory.ini ansible/<playbook>.yml -e env=<env> ansible-playbook \ -i deploy_configs/<env>/inventory.ini \ ansible/<playbook>.yml \ -e env=<env> \ -e image_tag=<image_tag> \ [-e harbor_project=<harbor_project_prefix>] ``` ### 私有 Harbor 镜像拉取(Docker login) Loading Loading @@ -123,7 +128,8 @@ ansible-playbook -i deploy_configs/${ENV}/inventory.ini ansible/update.yml -e en - 存放 Airflow 容器基础静态配置(如 `AIRFLOW_UID`)。 2) `deploy_configs/<env>/inventory.ini`(必需) - 定义部署拓扑(master/worker)、SSH 连接与关键变量(如 `deploy_dir`、`harbor_project`、`image_tag`、`jwt_secret`)。`worker` 组支持按主机单独设置 `worker_concurrency`,Ansible 在启动各 worker 时会把该值映射到容器内并发配置。 - 定义部署拓扑(master/worker)、SSH 连接与关键变量(如 `deploy_dir`、`registry_pull`、`harbor_registry`、`extra_hosts`、`jwt_secret`)。`worker` 组支持按主机单独设置 `worker_concurrency`,Ansible 在启动各 worker 时会把该值映射到容器内并发配置。 - 镜像 tag/仓库前缀默认由执行端的 `IMAGE_TAG/HARBOR_PROJECT` 传入(见上文);如需固化到环境配置,也可以在 inventory 的 `[all:vars]` 中设置 `image_tag/harbor_project` 覆盖。 3) `deploy_configs/<env>/variables.toml`(可选,但推荐) - `runtime`:唯一的共享运行时环境分组。部署时会同时导入 Airflow Variables(键名固定为 `runtime`)并渲染为 `deploy_configs/<env>/gateway.env`,分别供 DAG 任务执行阶段与 API Gateway 启动阶段使用。 Loading docker-celery-3.0.5/deploy_configs/csu/inventory.ini +6 −0 Original line number Diff line number Diff line Loading @@ -12,5 +12,11 @@ csu-worker1 ansible_host=192.168.25.14 ansible_user=root deploy_dir=/opt/csst-ai # The destination directory on remote nodes where the project will be deployed deploy_dir=/opt/csst-airflow registry_pull=false harbor_registry=harbor.csst.nao:10443 extra_hosts=harbor.csst.nao:10.80.1.21 # API Gateway 与 Airflow Webserver 通信所用的 JWT 密钥 (各环境可独立生成以保证安全) jwt_secret=eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ docker-celery-3.0.5/deploy_configs/csu/variables.toml +6 −5 Original line number Diff line number Diff line Loading @@ -5,15 +5,16 @@ CSST_DFS_ROOT = "/nfs/dfs" CCDS_SERVER_URL = "http://192.168.25.89:29000" CCDS_ROOT = "/nfs/ccds" CCDS_USER=USER CCDS_PASS=PASS CCDS_USER = "USER" CCDS_PASS = "PASS" HARBOR_PROJECT = "harbor.csst.nao:10443/csst" HARBOR_PROJECT = "csu-harbor.csst.nao:10443/csst" HARBOR_API_URL = "https://harbor.csst.nao:10443" HARBOR_API_URL = "https://csu-harbor.csst.nao:10443" HARBOR_API_PROJECT = "csst" HARBOR_API_VERIFY_SSL = false DUMPDATA=true DUMPDATA=false VERBOSE=true CSST_PORTAL_API_GATEWAY = "192.168.25.90:38501" Loading docker-celery-3.0.5/deploy_configs/p368/variables.toml +7 −0 Original line number Diff line number Diff line Loading @@ -5,6 +5,8 @@ CSST_DFS_ROOT = "/nfs/dfs" CCDS_SERVER_URL = "http://10.73.0.27:29000" CCDS_ROOT = "/nfs/ccds" CCDS_USER="USER" CCDS_PASS="PASS" HARBOR_PROJECT = "harbor.csst.nao:10443/csst" Loading @@ -12,6 +14,11 @@ HARBOR_API_URL = "https://harbor.csst.nao:10443" HARBOR_API_PROJECT = "csst" HARBOR_API_VERIFY_SSL = false DUMPDATA=false VERBOSE=true CSST_PORTAL_API_GATEWAY = "192.168.25.90:38501" [dag] default_retry_delay = 300 max_active_runs = 5 docker-celery-3.0.5/scripts/variables_toml_tool.py +111 −3 Original line number Diff line number Diff line Loading @@ -6,18 +6,126 @@ from __future__ import annotations import argparse import json import sys import tomllib from pathlib import Path from typing import Any def _strip_inline_comment(line: str) -> str: in_single = False in_double = False escaped = False out: list[str] = [] for ch in line: if escaped: out.append(ch) escaped = False continue if ch == "\\": out.append(ch) escaped = True continue if ch == "'" and not in_double: in_single = not in_single out.append(ch) continue if ch == '"' and not in_single: in_double = not in_double out.append(ch) continue if ch == "#" and not in_single and not in_double: break out.append(ch) return "".join(out).strip() def _unescape_basic_string(value: str) -> str: value = value.replace("\\\\", "\\") value = value.replace('\\"', '"') value = value.replace("\\n", "\n") value = value.replace("\\t", "\t") value = value.replace("\\r", "\r") return value def _parse_simple_value(raw: str) -> Any: raw = raw.strip() if not raw: return "" if raw.startswith('"') and raw.endswith('"') and len(raw) >= 2: return _unescape_basic_string(raw[1:-1]) if raw.startswith("'") and raw.endswith("'") and len(raw) >= 2: return raw[1:-1] lowered = raw.lower() if lowered == "true": return True if lowered == "false": return False if lowered == "null": return None try: return int(raw) except Exception: pass try: return float(raw) except Exception: pass return raw def _load_toml_fallback(file_path: Path) -> dict[str, Any]: current_section: dict[str, Any] | None = None data: dict[str, Any] = {} for raw_line in file_path.read_text(encoding="utf-8").splitlines(): line = _strip_inline_comment(raw_line).strip() if not line: continue if line.startswith("[") and line.endswith("]"): section_name = line[1:-1].strip() if not section_name: continue section = data.get(section_name) if not isinstance(section, dict): section = {} data[section_name] = section current_section = section continue if "=" not in line: continue key_raw, value_raw = line.split("=", 1) key = key_raw.strip() if not key: continue value = _parse_simple_value(value_raw) if current_section is None: data[key] = value else: current_section[key] = value return data def load_variables_toml(file_path: Path) -> dict[str, Any]: """读取 TOML;文件不存在时返回空配置,便于部署流程兜底。""" if not file_path.exists(): return {} try: import tomllib # type: ignore with file_path.open("rb") as file: data = tomllib.load(file) except ModuleNotFoundError: data = _load_toml_fallback(file_path) if not isinstance(data, dict): raise ValueError(f"variables.toml 顶层结构必须是 table: {file_path}") Loading Loading
README.md +9 −3 Original line number Diff line number Diff line Loading @@ -80,12 +80,17 @@ make ansible-down 说明: - `make pull` 只负责从 `required-images.txt` 拉取第三方镜像,不依赖 `IMAGE_TAG/BUILD_NUMBER/HARBOR_PROJECT`。 - `make push` 只负责把已有 tag 的镜像推送到仓库,不依赖 `BUILD_NUMBER`。 - `make ansible-*` 主要依赖 `ENV` 选择 inventory;镜像来源与 tag 由 `deploy_configs/<env>/inventory.ini` 中的 `harbor_project/image_tag` 控制。 - `make ansible-*` 主要依赖 `ENV` 选择 inventory;同时会把 `IMAGE_TAG`(必传,默认 `latest`)与 `HARBOR_PROJECT`(可选,镜像仓库前缀,如 `harbor.csst.nao:10443/csst/`)作为 Ansible extra vars 传入,用于决定 docker compose 使用的镜像 tag/前缀。 底层执行的命令形式为: ```bash ansible-playbook -i deploy_configs/<env>/inventory.ini ansible/<playbook>.yml -e env=<env> ansible-playbook \ -i deploy_configs/<env>/inventory.ini \ ansible/<playbook>.yml \ -e env=<env> \ -e image_tag=<image_tag> \ [-e harbor_project=<harbor_project_prefix>] ``` ### 私有 Harbor 镜像拉取(Docker login) Loading Loading @@ -123,7 +128,8 @@ ansible-playbook -i deploy_configs/${ENV}/inventory.ini ansible/update.yml -e en - 存放 Airflow 容器基础静态配置(如 `AIRFLOW_UID`)。 2) `deploy_configs/<env>/inventory.ini`(必需) - 定义部署拓扑(master/worker)、SSH 连接与关键变量(如 `deploy_dir`、`harbor_project`、`image_tag`、`jwt_secret`)。`worker` 组支持按主机单独设置 `worker_concurrency`,Ansible 在启动各 worker 时会把该值映射到容器内并发配置。 - 定义部署拓扑(master/worker)、SSH 连接与关键变量(如 `deploy_dir`、`registry_pull`、`harbor_registry`、`extra_hosts`、`jwt_secret`)。`worker` 组支持按主机单独设置 `worker_concurrency`,Ansible 在启动各 worker 时会把该值映射到容器内并发配置。 - 镜像 tag/仓库前缀默认由执行端的 `IMAGE_TAG/HARBOR_PROJECT` 传入(见上文);如需固化到环境配置,也可以在 inventory 的 `[all:vars]` 中设置 `image_tag/harbor_project` 覆盖。 3) `deploy_configs/<env>/variables.toml`(可选,但推荐) - `runtime`:唯一的共享运行时环境分组。部署时会同时导入 Airflow Variables(键名固定为 `runtime`)并渲染为 `deploy_configs/<env>/gateway.env`,分别供 DAG 任务执行阶段与 API Gateway 启动阶段使用。 Loading
docker-celery-3.0.5/deploy_configs/csu/inventory.ini +6 −0 Original line number Diff line number Diff line Loading @@ -12,5 +12,11 @@ csu-worker1 ansible_host=192.168.25.14 ansible_user=root deploy_dir=/opt/csst-ai # The destination directory on remote nodes where the project will be deployed deploy_dir=/opt/csst-airflow registry_pull=false harbor_registry=harbor.csst.nao:10443 extra_hosts=harbor.csst.nao:10.80.1.21 # API Gateway 与 Airflow Webserver 通信所用的 JWT 密钥 (各环境可独立生成以保证安全) jwt_secret=eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ
docker-celery-3.0.5/deploy_configs/csu/variables.toml +6 −5 Original line number Diff line number Diff line Loading @@ -5,15 +5,16 @@ CSST_DFS_ROOT = "/nfs/dfs" CCDS_SERVER_URL = "http://192.168.25.89:29000" CCDS_ROOT = "/nfs/ccds" CCDS_USER=USER CCDS_PASS=PASS CCDS_USER = "USER" CCDS_PASS = "PASS" HARBOR_PROJECT = "harbor.csst.nao:10443/csst" HARBOR_PROJECT = "csu-harbor.csst.nao:10443/csst" HARBOR_API_URL = "https://harbor.csst.nao:10443" HARBOR_API_URL = "https://csu-harbor.csst.nao:10443" HARBOR_API_PROJECT = "csst" HARBOR_API_VERIFY_SSL = false DUMPDATA=true DUMPDATA=false VERBOSE=true CSST_PORTAL_API_GATEWAY = "192.168.25.90:38501" Loading
docker-celery-3.0.5/deploy_configs/p368/variables.toml +7 −0 Original line number Diff line number Diff line Loading @@ -5,6 +5,8 @@ CSST_DFS_ROOT = "/nfs/dfs" CCDS_SERVER_URL = "http://10.73.0.27:29000" CCDS_ROOT = "/nfs/ccds" CCDS_USER="USER" CCDS_PASS="PASS" HARBOR_PROJECT = "harbor.csst.nao:10443/csst" Loading @@ -12,6 +14,11 @@ HARBOR_API_URL = "https://harbor.csst.nao:10443" HARBOR_API_PROJECT = "csst" HARBOR_API_VERIFY_SSL = false DUMPDATA=false VERBOSE=true CSST_PORTAL_API_GATEWAY = "192.168.25.90:38501" [dag] default_retry_delay = 300 max_active_runs = 5
docker-celery-3.0.5/scripts/variables_toml_tool.py +111 −3 Original line number Diff line number Diff line Loading @@ -6,18 +6,126 @@ from __future__ import annotations import argparse import json import sys import tomllib from pathlib import Path from typing import Any def _strip_inline_comment(line: str) -> str: in_single = False in_double = False escaped = False out: list[str] = [] for ch in line: if escaped: out.append(ch) escaped = False continue if ch == "\\": out.append(ch) escaped = True continue if ch == "'" and not in_double: in_single = not in_single out.append(ch) continue if ch == '"' and not in_single: in_double = not in_double out.append(ch) continue if ch == "#" and not in_single and not in_double: break out.append(ch) return "".join(out).strip() def _unescape_basic_string(value: str) -> str: value = value.replace("\\\\", "\\") value = value.replace('\\"', '"') value = value.replace("\\n", "\n") value = value.replace("\\t", "\t") value = value.replace("\\r", "\r") return value def _parse_simple_value(raw: str) -> Any: raw = raw.strip() if not raw: return "" if raw.startswith('"') and raw.endswith('"') and len(raw) >= 2: return _unescape_basic_string(raw[1:-1]) if raw.startswith("'") and raw.endswith("'") and len(raw) >= 2: return raw[1:-1] lowered = raw.lower() if lowered == "true": return True if lowered == "false": return False if lowered == "null": return None try: return int(raw) except Exception: pass try: return float(raw) except Exception: pass return raw def _load_toml_fallback(file_path: Path) -> dict[str, Any]: current_section: dict[str, Any] | None = None data: dict[str, Any] = {} for raw_line in file_path.read_text(encoding="utf-8").splitlines(): line = _strip_inline_comment(raw_line).strip() if not line: continue if line.startswith("[") and line.endswith("]"): section_name = line[1:-1].strip() if not section_name: continue section = data.get(section_name) if not isinstance(section, dict): section = {} data[section_name] = section current_section = section continue if "=" not in line: continue key_raw, value_raw = line.split("=", 1) key = key_raw.strip() if not key: continue value = _parse_simple_value(value_raw) if current_section is None: data[key] = value else: current_section[key] = value return data def load_variables_toml(file_path: Path) -> dict[str, Any]: """读取 TOML;文件不存在时返回空配置,便于部署流程兜底。""" if not file_path.exists(): return {} try: import tomllib # type: ignore with file_path.open("rb") as file: data = tomllib.load(file) except ModuleNotFoundError: data = _load_toml_fallback(file_path) if not isinstance(data, dict): raise ValueError(f"variables.toml 顶层结构必须是 table: {file_path}") Loading