feat(deploy): 添加分阶段滚动更新与worker运行时信息收集

SUBDIR ?= docker-celery-3.0.5

.PHONY: help list-envs pull build push clean all \

        ansible-deploy ansible-update ansible-down ansible-check ansible-sync-dags \

        ansible-deploy ansible-update ansible-update-master ansible-update-worker \

        ansible-down ansible-check ansible-sync-dags \

        viewer-test viewer-test-down update-code

help:

	@$(MAKE) -C $(SUBDIR) help

list-envs pull build push clean all \

ansible-deploy ansible-update ansible-down ansible-check ansible-sync-dags \

ansible-deploy ansible-update ansible-update-master ansible-update-worker \

ansible-down ansible-check ansible-sync-dags \

viewer-test viewer-test-down update-code:

	@$(MAKE) -C $(SUBDIR) $@

make ansible-update

```

说明：

- `make ansible-update` 会执行完整更新流程：同步代码、更新 `master`、滚动更新 `worker`，最后重启 `flower/api-gateway/task-portal`。

- 如果只想更新控制面，请使用 `make ansible-update-master`。

- 如果只想更新执行节点，请使用 `make ansible-update-worker`。

仅部署“已经构建好的镜像”（Harbor 中已存在对应 tag）：无需执行 `make build`，直接部署即可。

```bash

make ansible-update

```

`tools/deploy_csu.sh` 保持为简单脚本，但内部执行的是可靠的两段式流程：`build -> push -> clean -> ansible-update-master -> ansible-update-worker`。

- 会先更新 `master`（包括 `portal/api-gateway/flower` 等控制面容器），再滚动更新 `worker`。

- 脚本通过预先 `export` 的环境变量驱动 `make`，不在 `make` 命令行内联传参。

- 脚本固定导出 `registry_pull=true`，即使测试阶段重复使用 `BUILD_NUMBER=100000`，远端也会强制拉取最新镜像。

- 脚本固定导出 `allow_busy_worker_update=false`，busy worker 会被跳过而不是强制中断。

提示：仓库根目录的 `Makefile` 是 wrapper，默认转发到 `docker-celery-3.0.5/` 目录执行。如果未来目录变更，可通过 `SUBDIR=...` 指定。

## 前置条件

make ansible-check

make ansible-deploy

make ansible-update

make ansible-update-master

make ansible-update-worker

make ansible-sync-dags

make ansible-down

```

- `make push` 会把镜像推送到仓库，镜像 tag 使用 `BUILD_NUMBER`（默认 `latest`）。

- `make clean` 会统一清理本地构建镜像、dangling images、builder cache，以及其他未使用镜像。

- `make ansible-*` 主要依赖 `ENV` 选择 inventory；同时会把 `BUILD_NUMBER`（镜像 tag，默认 `latest`）与 `HARBOR_PROJECT`（可选，镜像仓库前缀，如 `harbor.csst.nao:10443/csst/`）作为 Ansible extra vars 传入，用于决定 docker compose 使用的镜像 tag/前缀。

- `make ansible-update`：更新 `master + worker`。

- `make ansible-update-master`：只更新 `master`。

- `make ansible-update-worker`：只更新 `worker`，并在 `serial: 1` 的滚动流程中跳过 busy worker。

底层执行的命令形式为：

- Docker 权限：确保执行用户已加入 `docker` 组或具备 sudo/root。

- 端口冲突：检查 `38501/38000/38080/33000/35601/35555/39200/35432/36379/39090` 是否被占用。

- Worker 不显示：优先检查部署环境变量与 worker 启动参数是否正确；必要时执行 `make ansible-update ENV=<env>` 触发重启发现。

- Worker 不显示：优先检查部署环境变量与 worker 启动参数是否正确；必要时执行 `make ansible-update-worker ENV=<env>` 触发 worker 侧滚动更新，并由后置步骤重启 `flower/api-gateway/task-portal` 重新发现。

- Workers 页面 `Build` 列显示的是每台 worker 当前运行容器的真实镜像 tag；如果部分 busy worker 被跳过更新，页面可能会同时出现不同 build，这是预期行为。

- ES Unhealthy：常见为数据卷权限问题，可尝试修复 `volumes/es_data` 的属主权限后重启相关容器。

- 查看日志：在部署目录执行 `docker compose logs -f`。

	@echo "  make all	 - Execute clean, pull, build, and push sequentially"

	@echo "  make ansible-deploy - Deploy to environment (e.g. make ansible-deploy ENV=p368)"

	@echo "  make ansible-update - Update environment (e.g. make ansible-update ENV=p368)"

	@echo "  make ansible-update-master - Update master-only services (e.g. make ansible-update-master ENV=csu)"

	@echo "  make ansible-update-worker - Update workers safely (e.g. make ansible-update-worker ENV=csu)"

	@echo "  make ansible-down   - Tear down environment (e.g. make ansible-down ENV=p368)"

	@echo "  make ansible-check  - Run check playbook (e.g. make ansible-check ENV=p368)"

	@echo "  make ansible-sync-dags - Sync DAGs to environment (e.g. make ansible-sync-dags ENV=p368)"

	echo "EFFECTIVE_HARBOR_PROJECT=$$effective_harbor_project"

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/update.yml $(ANSIBLE_EXTRA_VARS)

ansible-update-master:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-update-master ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/update.yml $(ANSIBLE_EXTRA_VARS) --tags master_update

ansible-update-worker:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-update-worker ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/update.yml $(ANSIBLE_EXTRA_VARS) --tags worker_update,post_worker_update

ansible-down:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-down ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

        path: "{{ _deploy_dir }}/deploy_configs/{{ _env }}"

        state: directory

    - name: Render worker_specs.json to remote

      template:

        src: templates/worker-specs.json.j2

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/worker_specs.json"

    - name: Copy airflow.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/airflow.env"

      args:

        chdir: "{{ _deploy_dir }}"

    - name: Inspect running airflow-worker image on worker

      shell: |

        COMPOSE_FILES="-f docker-compose.yaml"

        if [ -f deploy_configs/{{ _env }}/docker-compose.extra-hosts.override.yml ]; then

          COMPOSE_FILES="$COMPOSE_FILES -f deploy_configs/{{ _env }}/docker-compose.extra-hosts.override.yml"

        fi

        cid=$({{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker ps -q airflow-worker 2>/dev/null || true)

        if [ -z "$cid" ]; then

          printf '%s\n' '{"image_ref": "", "image_id": ""}'

          exit 0

        fi

        image_ref=$(docker inspect "$cid" --format '{% raw %}{{.Config.Image}}{% endraw %}')

        image_id=$(docker inspect "$cid" --format '{% raw %}{{.Image}}{% endraw %}')

        python3 -c 'import json, sys; print(json.dumps({"image_ref": sys.argv[1], "image_id": sys.argv[2]}))' "$image_ref" "$image_id"

      args:

        chdir: "{{ _deploy_dir }}"

      register: worker_runtime_inspect

      changed_when: false

    - name: Record worker runtime image metadata

      set_fact:

        worker_actual_image_ref: "{{ (worker_runtime_inspect.stdout | from_json).image_ref | default('') }}"

        worker_actual_image_id: "{{ (worker_runtime_inspect.stdout | from_json).image_id | default('') }}"

- name: Post-Deploy Master Node Services (Ensure worker discovery)

  hosts: master

  become: yes

    _extra_hosts_list: "{{ (extra_hosts | default('') ).split(',') | map('trim') | reject('equalto','') | list }}"

    _build_number: "{{ build_number | default('latest') }}"

  tasks:

    - name: Render worker_runtime.json to remote

      template:

        src: templates/worker-runtime.json.j2

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/worker_runtime.json"

    - name: Restart Flower and API Gateway to ensure they detect new workers

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

{

{% for host in groups['worker'] | default([]) %}

  "{{ host }}": {

    "image_ref": "{{ hostvars[host].worker_actual_image_ref | default('') }}",

    "image_id": "{{ hostvars[host].worker_actual_image_id | default('') }}"

  }{% if not loop.last %},{% endif %}

{% endfor %}

}