Commit 9587759c authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

feat: 添加worker状态监控并优化部署配置

parent b9ba9e34

.gitignore

+1 −1
Original line number Diff line number Diff line
@@ -63,4 +63,4 @@ data/ 
# Generated during Ansible variable import

docker-celery-3.0.5/api_gateway/ccds/

docker-celery-3.0.5/api_gateway/csst-dfs-client/

#external/csst-dag
 
external/csst-dag/

Makefile

+3 −2
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ SUBDIR ?= docker-celery-3.0.5 


.PHONY: help list-envs pull build push clean all \

        ansible-deploy ansible-update ansible-update-master ansible-update-worker \

        ansible-down ansible-check ansible-sync-dags \

        ansible-start ansible-stop ansible-kill ansible-down ansible-check ansible-sync-dags \

        viewer-test viewer-test-down update-code



help:
@@ -13,11 +13,12 @@ help: 
	@echo "Common usage:"

	@echo "  make build BUILD_NUMBER=100000 HARBOR_PROJECT=harbor.csst.nao:10443/csst/"

	@echo "  make ansible-update ENV=p368"

	@echo "  make ansible-stop ENV=csu"

	@echo ""

	@$(MAKE) -C $(SUBDIR) help



list-envs pull build push clean all \

ansible-deploy ansible-update ansible-update-master ansible-update-worker \

ansible-down ansible-check ansible-sync-dags \

ansible-start ansible-stop ansible-kill ansible-down ansible-check ansible-sync-dags \

viewer-test viewer-test-down update-code:

	@$(MAKE) -C $(SUBDIR) $@

README.md

+53 −0
Original line number Diff line number Diff line
@@ -105,6 +105,9 @@ make ansible-deploy 
make ansible-update

make ansible-update-master

make ansible-update-worker

make ansible-start

make ansible-stop

make ansible-kill

make ansible-sync-dags

make ansible-down

```
@@ -117,6 +120,56 @@ make ansible-down 
- `make ansible-update`：更新 `master + worker`。

- `make ansible-update-master`：只更新 `master`。

- `make ansible-update-worker`：只更新 `worker`，并在 `serial: 1` 的滚动流程中跳过 busy worker。

- `make ansible-start`：重新启动现有服务，不重新同步代码。

- `make ansible-stop`：优雅停止服务但保留容器。

- `make ansible-kill`：强制终止服务。

- `make ansible-down`：拆除整套服务（容器/网络）。



### 什么时候用哪个



- 首次部署环境：用 `make ansible-deploy ENV=csu`

- 平时改了代码、前端、DAG、配置后上线：用 `make ansible-update ENV=csu`

- 只想更新 worker：用 `make ansible-update-worker ENV=csu`

- 只想更新 master：用 `make ansible-update-master ENV=csu`



### 典型运维场景



临时停服务但保留容器：



```bash

export ENV=csu

make ansible-stop

```



停机后重新启动现有服务：



```bash

export ENV=csu

make ansible-start

```



容器卡死，需要强制终止：



```bash

export ENV=csu

make ansible-kill

```



做代码/镜像更新并滚动重启：



```bash

export ENV=csu

export BUILD_NUMBER=100000

make ansible-update

```



整套卸载并重新完整拉起：



```bash

export ENV=csu

make ansible-down

make ansible-deploy

```



底层执行的命令形式为：

docker-celery-3.0.5/Makefile

+31 −9
Original line number Diff line number Diff line
@@ -17,11 +17,12 @@ REQ_FILE := required-images.txt 
TARGET_AIRFLOW := $(HARBOR_PROJECT)csst-airflow:$(BUILD_NUMBER)

TARGET_API_GATEWAY := $(HARBOR_PROJECT)csst-airflow-api-gateway:$(BUILD_NUMBER)

TARGET_TASK_PORTAL := $(HARBOR_PROJECT)csst-airflow-task-portal:$(BUILD_NUMBER)

TARGET_WORKER_STATS_AGENT := $(HARBOR_PROJECT)csst-airflow-worker-stats-agent:$(BUILD_NUMBER)

TARGET_JS9 := $(HARBOR_PROJECT)csst-airflow-js9:$(BUILD_NUMBER)



CUSTOM_IMAGE_COUNT := 4

CUSTOM_IMAGE_COUNT := 5

ifeq ($(strip $(SKIP_JS9)),1)

CUSTOM_IMAGE_COUNT := 3

CUSTOM_IMAGE_COUNT := 4

endif



ANSIBLE_EXTRA_VARS := -e env=$(ENV) -e build_number=$(BUILD_NUMBER)
@@ -58,13 +59,16 @@ help: 
	@echo "  make ansible-update - Update environment (e.g. make ansible-update ENV=p368)"

	@echo "  make ansible-update-master - Update master-only services (e.g. make ansible-update-master ENV=csu)"

	@echo "  make ansible-update-worker - Update workers safely (e.g. make ansible-update-worker ENV=csu)"

	@echo "  make ansible-start  - Start existing environment services"

	@echo "  make ansible-stop   - Stop environment services without removing containers"

	@echo "  make ansible-kill   - Force kill environment services"

	@echo "  make ansible-down   - Tear down environment (e.g. make ansible-down ENV=p368)"

	@echo "  make ansible-check  - Run check playbook (e.g. make ansible-check ENV=p368)"

	@echo "  make ansible-sync-dags - Sync DAGs to environment (e.g. make ansible-sync-dags ENV=p368)"

	@echo ""

	@echo "Images to be processed:"

	@echo "  Third-party: postgres, redis, elasticsearch, kibana, filebeat, prometheus, grafana, cadvisor, node-exporter"

	@echo "  Custom:      csst-airflow, csst-airflow-api-gateway, csst-airflow-task-portal (and optional csst-airflow-js9)"

	@echo "  Custom:      csst-airflow, csst-airflow-api-gateway, csst-airflow-task-portal, csst-airflow-worker-stats-agent (and optional csst-airflow-js9)"

	@echo "======================================================================"



list-envs:
@@ -117,10 +121,12 @@ build: 
				rsync -a --delete --exclude ".git" --exclude "__pycache__" ../external/csst-dag/ ./api_gateway/csst-dag/; \

		fi; \

	docker build $(DOCKER_BUILD_FLAGS) $(API_GATEWAY_BUILD_ARGS) -t $(TARGET_API_GATEWAY) ./api_gateway; \

	echo "[$$(($$total+3))/$${total_imgs}] Building Task Portal Image ($(TARGET_TASK_PORTAL)) with BUILD_NUMBER=$(BUILD_NUMBER)..."; \

	docker build $(DOCKER_BUILD_FLAGS) --build-arg VITE_BUILD_NUMBER=$(BUILD_NUMBER) --build-arg VITE_BUILD_TIME="$(shell date '+%Y-%m-%d %H:%M:%S')" -t $(TARGET_TASK_PORTAL) ./frontend_vue; \

	echo "[$$(($$total+3))/$${total_imgs}] Building Worker Stats Agent Image ($(TARGET_WORKER_STATS_AGENT))..."; \

	docker build $(DOCKER_BUILD_FLAGS) -t $(TARGET_WORKER_STATS_AGENT) ./worker_stats_agent; \

	echo "[$$(($$total+4))/$${total_imgs}] Building Task Portal Image ($(TARGET_TASK_PORTAL)) with BUILD_NUMBER=$(BUILD_NUMBER)..."; \

	docker build $(DOCKER_BUILD_FLAGS) --build-arg VITE_BUILD_NUMBER=$(BUILD_NUMBER) --build-arg VITE_BUILD_TIME="$(shell TZ=Asia/Shanghai date '+%Y-%m-%d %H:%M:%S %Z')" -t $(TARGET_TASK_PORTAL) ./frontend_vue; \

	if [ "$(SKIP_JS9)" != "1" ]; then \

		echo "[$$(($$total+4))/$${total_imgs}] Building JS9 Image ($(TARGET_JS9))..."; \

		echo "[$$(($$total+5))/$${total_imgs}] Building JS9 Image ($(TARGET_JS9))..."; \

		if [ -d "../external/js9" ]; then \

				mkdir -p ./external/js9; \

				rsync -a --delete --exclude ".git" --exclude "__pycache__" ../external/js9/ ./external/js9/; \
@@ -146,9 +152,10 @@ push: 
	done; \

	echo "[$$(($$total+1))/$${total_imgs}] Pushing $(TARGET_AIRFLOW)..." && docker push $(TARGET_AIRFLOW); \

	echo "[$$(($$total+2))/$${total_imgs}] Pushing $(TARGET_API_GATEWAY)..." && docker push $(TARGET_API_GATEWAY); \

	echo "[$$(($$total+3))/$${total_imgs}] Pushing $(TARGET_TASK_PORTAL)..." && docker push $(TARGET_TASK_PORTAL); \

	echo "[$$(($$total+3))/$${total_imgs}] Pushing $(TARGET_WORKER_STATS_AGENT)..." && docker push $(TARGET_WORKER_STATS_AGENT); \

	echo "[$$(($$total+4))/$${total_imgs}] Pushing $(TARGET_TASK_PORTAL)..." && docker push $(TARGET_TASK_PORTAL); \

	if [ "$(SKIP_JS9)" != "1" ]; then \

		echo "[$$(($$total+4))/$${total_imgs}] Pushing $(TARGET_JS9)..." && docker push $(TARGET_JS9); \

		echo "[$$(($$total+5))/$${total_imgs}] Pushing $(TARGET_JS9)..." && docker push $(TARGET_JS9); \

	fi; \

	echo "All $${total_imgs} images pushed successfully to $(HARBOR_PROJECT) with tag $(BUILD_NUMBER)."
@@ -157,7 +164,7 @@ all: clean pull build push 
clean:

	@echo "Cleaning up custom-built images..."

	@set -e; \

	ids=$$(docker images --format '{{.Repository}}:{{.Tag}} {{.ID}}' | awk '$$1 ~ /(csst-airflow|api-gateway|task-portal|js9)/ {print $$2}' | sort -u); \

	ids=$$(docker images --format '{{.Repository}}:{{.Tag}} {{.ID}}' | awk '$$1 ~ /(csst-airflow|api-gateway|task-portal|worker-stats-agent|js9)/ {print $$2}' | sort -u); \

	for id in $$ids; do \

		if docker ps -q --filter ancestor=$$id | grep -q .; then \

			continue; \
@@ -214,6 +221,21 @@ ansible-down: 
	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/down.yml $(ANSIBLE_EXTRA_VARS)



ansible-stop:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-stop ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/stop.yml $(ANSIBLE_EXTRA_VARS)



ansible-start:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-start ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/start.yml $(ANSIBLE_EXTRA_VARS)



ansible-kill:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-kill ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/kill.yml $(ANSIBLE_EXTRA_VARS)



ansible-check:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-check ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

docker-celery-3.0.5/ansible/deploy.yml

+60 −8
Original line number Diff line number Diff line
@@ -93,14 +93,31 @@ 
        path: "{{ _deploy_dir }}/deploy_configs/{{ _env }}"

        state: directory



    - name: Ensure pipeline output root exists on master

      file:

        path: "{{ pipeline_output_root | default('/opt/csst-airflow-pipeline-outputs') }}"

        state: directory

        owner: "50000"

        group: "0"

        mode: "0775"



    - name: Ensure Elasticsearch data directory is writable by container user

      file:

        path: "{{ _deploy_dir }}/volumes/es_data"

        state: directory

        owner: "1000"

        group: "0"

        mode: "0775"

        recurse: yes



    - name: Render worker_specs.json to remote

      template:

        src: templates/worker-specs.json.j2

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/worker_specs.json"



    - name: Copy airflow.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/airflow.env"

    - name: Render airflow.env to remote

      template:

        src: templates/airflow.env.j2

        dest: "{{ _deploy_dir }}/deploy_configs/airflow.env"



    - name: Copy variables.toml to remote
@@ -149,6 +166,7 @@ 


    - name: Start Master services (postgres, redis, elasticsearch, airflow core, api, etc.)

      shell: |

        set -e

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export BUILD_NUMBER="{{ _build_number }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"
@@ -178,6 +196,7 @@ 
        docker rmi -f {{ harbor_project | default('') }}csst-airflow-task-portal:{{ _build_number }} || true

        docker rmi -f {{ harbor_project | default('') }}csst-airflow-js9:{{ _build_number }} || true

        docker rmi -f {{ harbor_project | default('') }}csst-airflow:{{ _build_number }} || true

        docker rmi -f {{ harbor_project | default('') }}csst-airflow-worker-stats-agent:{{ _build_number }} || true

        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master pull

        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master up -d --remove-orphans

        ensure_tag() {
@@ -232,9 +251,17 @@ 
        path: "{{ _deploy_dir }}/deploy_configs/{{ _env }}"

        state: directory



    - name: Copy airflow.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/airflow.env"

    - name: Ensure pipeline output root exists on worker

      file:

        path: "{{ pipeline_output_root | default('/opt/csst-airflow-pipeline-outputs') }}"

        state: directory

        owner: "50000"

        group: "0"

        mode: "0775"



    - name: Render airflow.env to remote

      template:

        src: templates/airflow.env.j2

        dest: "{{ _deploy_dir }}/deploy_configs/airflow.env"



    - name: Copy variables.toml to worker remote
@@ -257,6 +284,25 @@ 
        extra_hosts_list: "{{ _extra_hosts_list }}"

      when: _extra_hosts_list | length > 0



    - name: Inspect docker socket gid on worker

      stat:

        path: /var/run/docker.sock

      register: docker_sock_stat



    - name: Render docker compose docker sock override on worker

      template:

        src: templates/docker-compose.docker-sock.override.yml.j2

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/docker-compose.docker-sock.override.yml"

      vars:

        docker_sock_gid: "{{ docker_sock_stat.stat.gid }}"

      when: docker_sock_stat.stat.exists | default(false)



    - name: Remove docker sock override when socket is absent

      file:

        path: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/docker-compose.docker-sock.override.yml"

        state: absent

      when: not (docker_sock_stat.stat.exists | default(false))



    - name: Remove legacy dag.env on worker remote

      file:

        path: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/dag.env"
@@ -274,9 +320,11 @@ 
        - "{{ _deploy_dir }}/dags"

        - "{{ _deploy_dir }}/volumes/plugins"

        - "{{ _deploy_dir }}/volumes/config"

        - "{{ pipeline_output_root | default('/opt/csst-airflow-pipeline-outputs') }}"



    - name: Start Worker services (celery-worker, filebeat, monitoring)

      shell: |

        set -e

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export BUILD_NUMBER="{{ _build_number }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"
@@ -289,13 +337,16 @@ 
        if [ -f deploy_configs/{{ _env }}/docker-compose.extra-hosts.override.yml ]; then

          COMPOSE_FILES="$COMPOSE_FILES -f deploy_configs/{{ _env }}/docker-compose.extra-hosts.override.yml"

        fi

        if [ -f deploy_configs/{{ _env }}/docker-compose.docker-sock.override.yml ]; then

          COMPOSE_FILES="$COMPOSE_FILES -f deploy_configs/{{ _env }}/docker-compose.docker-sock.override.yml"

        fi



        docker rmi -f {{ harbor_project | default('') }}filebeat:{{ _build_number }} || true

        docker rmi -f {{ harbor_project | default('') }}cadvisor:{{ _build_number }} || true

        docker rmi -f {{ harbor_project | default('') }}node-exporter:{{ _build_number }} || true

        docker rmi -f {{ harbor_project | default('') }}csst-airflow:{{ _build_number }} || true

        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker pull

        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker up -d --remove-orphans --force-recreate

        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker up -d --remove-orphans --force-recreate airflow-worker worker-stats-agent filebeat cadvisor node-exporter

        ensure_tag() {

          svc="$1"

          cid=$({{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker ps -q "$svc" 2>/dev/null || true)
@@ -305,7 +356,7 @@ 
            docker image inspect "$img_ref" >/dev/null 2>&1 || docker image tag "$img_id" "$img_ref"

          fi

        }

        for svc in airflow-worker filebeat cadvisor node-exporter; do

        for svc in airflow-worker worker-stats-agent filebeat cadvisor node-exporter; do

          ensure_tag "$svc"

        done

        master_left=$({{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master ps -q || true)
@@ -357,6 +408,7 @@ 


    - name: Restart Flower and API Gateway to ensure they detect new workers

      shell: |

        set -e

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export BUILD_NUMBER="{{ _build_number }}"

        export ENV="{{ _env }}"