Commit 892e0326 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

feat(deploy): 重构部署配置与镜像管理流程

parent 1a481edd

.gitignore

+4 −0
Original line number Diff line number Diff line
@@ -55,3 +55,7 @@ cookies.txt 
# Tests

tests/

tmp/



# External files

external/

README.md

+1 −1
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ export HARBOR_PROJECT=harbor.csst.nao:10443/csst/ 
# 设置构建镜像的版本号

export IMAGE_TAG=test

# 设置内部构建号 (将展示在 Portal 前端)，如果不设置将默认使用 IMAGE_TAG 的值

export BUILD_NUMBER=123456

export BUILD_NUMBER=test



# ----------------------------------------

# 【方案二：本地单机开发测试环境】

docker-celery-3.0.5/Makefile

+105 −49
Original line number Diff line number Diff line 


# 默认环境（可被传入的 ENV 参数覆盖，例如 make up-master ENV=csu）

ENV ?= csu

ENV_FILE := envs/$(ENV).env



# 动态检测 docker compose 还是 docker-compose

DOCKER_COMPOSE := $(shell if docker compose version >/dev/null 2>&1; then echo "docker compose"; elif command -v docker-compose >/dev/null 2>&1; then echo "docker-compose"; else echo "echo 'No docker compose found'; exit 1"; fi)

# 镜像版本和构建相关

IMAGE_TAG ?= latest

BUILD_NUMBER ?= $(IMAGE_TAG)

HARBOR_PROJECT ?= 



# 第三方依赖镜像清单文件

REQ_FILE := required-images.txt



# 业务自定义镜像

TARGET_AIRFLOW := $(HARBOR_PROJECT)csst-airflow:$(IMAGE_TAG)

TARGET_API_GATEWAY := $(HARBOR_PROJECT)csst-airflow-api-gateway:$(IMAGE_TAG)

TARGET_TASK_PORTAL := $(HARBOR_PROJECT)csst-airflow-task-portal:$(IMAGE_TAG)



help:

	@echo "======================================================================"

	@echo "                 CSST Airflow Operations & Delivery                   "

	@echo "======================================================================"

	@echo "Available make commands:"

	@echo "  make help        - Show this help message"

	@echo "  make pull        - Pull all required official third-party images"

	@echo "  make build       - Pull, retag third-party images, and build custom images"

	@echo "  make push        - Push all prepared images to the specified Harbor registry"

	@echo "  make clean       - Remove all custom built images (Airflow, API Gateway, Portal)"

	@echo "  make all         - Execute clean, pull, build, and push sequentially"

	@echo ""

	@echo "Images to be processed:"

	@echo "  Third-party: postgres, redis, elasticsearch, kibana, filebeat, prometheus, grafana, cadvisor, node-exporter"

	@echo "  Custom:      csst-airflow, csst-airflow-api-gateway, csst-airflow-task-portal"

	@echo "======================================================================"



list-envs:

	@echo "Available environments (Usage: make up-master ENV=<name>):"

	@ls envs/*.env 2>/dev/null | sed 's|envs/||' | sed 's|.env||' | sed 's/^/  - /' || echo "  No environment files found."



all: down pull mkdir fix-permission init up



pull:

	git pull



mkdir:

	mkdir -p ./dags ./volumes/logs ./volumes/plugins ./volumes/config ./volumes/pg_data ./volumes/es_data



fix-permission:

	sudo chown -R 50000:0 ./volumes/es_data



rmdir:

	rm -rf ./volumes



init:

	$(DOCKER_COMPOSE) --env-file $(ENV_FILE) up airflow-init



up:

	$(DOCKER_COMPOSE) --env-file $(ENV_FILE) up -d



check-env:

	@if [ ! -f $(ENV_FILE) ]; then echo "Error: Environment file $(ENV_FILE) does not exist!"; exit 1; fi



up-master: check-env

	$(DOCKER_COMPOSE) -f docker-compose.yaml --env-file $(ENV_FILE) --profile flower --profile master up -d --force-recreate



up-worker: check-env

	$(DOCKER_COMPOSE) -f docker-compose.yaml --env-file $(ENV_FILE) --profile worker up -d --force-recreate



down:

	$(DOCKER_COMPOSE) --env-file $(ENV_FILE) down



migrate:

	$(DOCKER_COMPOSE) --env-file $(ENV_FILE) run --rm airflow-init airflow db upgrade

	@echo "Available environments (Configured in deploy_configs/):"

	@ls -d deploy_configs/*/ 2>/dev/null | sed 's|deploy_configs/||' | sed 's|/||' | sed 's/^/  - /' || echo "  No environments found."



build:

	@if [ ! -f $(REQ_FILE) ]; then echo "Error: $(REQ_FILE) not found"; exit 1; fi

	@total=$$(grep -v '^#' $(REQ_FILE) | grep -v '^$$' | wc -l); \

	echo "Pulling, retagging and building all $$(($$total+3)) images..."; \

	count=1; \

	for img in $$(grep -v '^#' $(REQ_FILE) | grep -v '^$$'); do \

		basename="$${img##*/}"; \

		name="$${basename%%:*}"; \

		target="$(HARBOR_PROJECT)$${name}:$(IMAGE_TAG)"; \

		if docker image inspect $${img} > /dev/null 2>&1; then \

			echo "[$${count}/$$(($$total+3))] Processing $${img} -> $${target} (Skipping pull, already exists)..."; \

		else \

			echo "[$${count}/$$(($$total+3))] Processing $${img} -> $${target}..."; \

			(docker pull $${img} > /dev/null 2>&1 || true); \

		fi; \

		if [ "$${img}" != "$${target}" ]; then \

			(docker tag $${img} $${target} > /dev/null 2>&1 || true); \

		fi; \

		count=$$((count+1)); \

	done; \

	echo "[$$(($$total+1))/$$(($$total+3))] Building Airflow Custom Image ($(TARGET_AIRFLOW))..."; \

	docker build -q -t $(TARGET_AIRFLOW) .; \

	echo "[$$(($$total+2))/$$(($$total+3))] Building API Gateway Image ($(TARGET_API_GATEWAY))..."; \

	docker build -q -t $(TARGET_API_GATEWAY) ./api_gateway; \

	echo "[$$(($$total+3))/$$(($$total+3))] Building Task Portal Image ($(TARGET_TASK_PORTAL)) with BUILD_NUMBER=$(BUILD_NUMBER)..."; \

	docker build -q --build-arg VITE_BUILD_NUMBER=$(BUILD_NUMBER) -t $(TARGET_TASK_PORTAL) ./frontend_vue; \

	echo "All $$(($$total+3)) images prepared successfully."



push:

	@if [ -z "$(HARBOR_PROJECT)" ]; then echo "Error: HARBOR_PROJECT is empty. Please specify a registry prefix to push. Example: make push HARBOR_PROJECT=csu-harbor.csst.nao:10443/csst/"; exit 1; fi

	@total=$$(grep -v '^#' $(REQ_FILE) | grep -v '^$$' | wc -l); \

	echo "Pushing all $$(($$total+3)) images to $(HARBOR_PROJECT)..."; \

	count=1; \

	for img in $$(grep -v '^#' $(REQ_FILE) | grep -v '^$$'); do \

		basename="$${img##*/}"; \

		name="$${basename%%:*}"; \

		target="$(HARBOR_PROJECT)$${name}:$(IMAGE_TAG)"; \

		echo "[$${count}/$$(($$total+3))] Pushing $${target}..."; \

		docker push $${target}; \

		count=$$((count+1)); \

	done; \

	echo "[$$(($$total+1))/$$(($$total+3))] Pushing $(TARGET_AIRFLOW)..." && docker push $(TARGET_AIRFLOW); \

	echo "[$$(($$total+2))/$$(($$total+3))] Pushing $(TARGET_API_GATEWAY)..." && docker push $(TARGET_API_GATEWAY); \

	echo "[$$(($$total+3))/$$(($$total+3))] Pushing $(TARGET_TASK_PORTAL)..." && docker push $(TARGET_TASK_PORTAL); \

	echo "All $$(($$total+3)) images pushed successfully to $(HARBOR_PROJECT) with tag $(IMAGE_TAG)."



all: clean pull build push



clean:

	# 停止服务并清理旧数据

	$(DOCKER_COMPOSE) --env-file $(ENV_FILE) rm -sf

	@echo "Cleaning up custom-built images..."

	@docker images | grep -E 'csst-airflow|api-gateway|task-portal' | awk '{print $$3}' | xargs -r docker rmi -f || true

	@echo "Clean completed."



ps:

	$(DOCKER_COMPOSE) --env-file $(ENV_FILE) ps

update-code:

	git pull



restart:

	$(DOCKER_COMPOSE) --env-file $(ENV_FILE) restart
 
pull:

	@total=$$(grep -v '^#' $(REQ_FILE) | grep -v '^$$' | wc -l); \

	echo "Required official images to pull ($$total in total):"; \

	count=1; \

	for img in $$(grep -v '^#' $(REQ_FILE) | grep -v '^$$'); do \

		echo "  [$$count/$$total] $$img"; \

		count=$$((count+1)); \

	done

	@echo "----------------------------------------"

	@total=$$(grep -v '^#' $(REQ_FILE) | grep -v '^$$' | wc -l); \

	count=1; \

	for img in $$(grep -v '^#' $(REQ_FILE) | grep -v '^$$'); do \

		if docker image inspect $${img} > /dev/null 2>&1; then \

			echo "[$${count}/$${total}] Skipping $${img} (Already exists locally)..."; \

		else \

			echo "[$${count}/$${total}] Pulling $${img}..."; \

			(docker pull $${img} || true); \

		fi; \

		count=$$((count+1)); \

	done

	@echo "All official images pull requests completed."

docker-celery-3.0.5/ansible/check.yml

0 → 100644
+40 −0
Original line number Diff line number Diff line 
---

- name: Pre-flight Environment and Connectivity Check

  hosts: all

  become: yes

  vars:

    _env: "{{ env | default('p368') }}"

    _deploy_dir: "{{ deploy_dir | default('/opt/csst-airflow') }}"

    

  tasks:

    - name: 1. Verify SSH connectivity and Ping

      ping:



    - name: 2. Check if Docker is installed and running

      command: docker info

      register: docker_check

      changed_when: false

      failed_when: docker_check.rc != 0



    - name: 3. Check if Docker Compose plugin is available

      shell: docker compose version || docker-compose --version

      register: docker_compose_check

      changed_when: false

      failed_when: docker_compose_check.rc != 0



    - name: 4. Ensure deployment directory base path exists or can be created

      file:

        path: "{{ _deploy_dir | dirname }}"

        state: directory



    - name: 5. Check Disk Space (Warn if less than 10GB free in deploy root)

      shell: df -BG {{ _deploy_dir | dirname }} | awk 'NR==2 {print $4}' | sed 's/G//'

      register: disk_space

      changed_when: false



    - name: Print Node Readiness Status

      debug:

        msg: 

          - "Node {{ inventory_hostname }} is READY for deployment."

          - "Docker and Compose are functional."

          - "Free Disk Space: {{ disk_space.stdout }}GB"

docker-celery-3.0.5/ansible/deploy.yml

+76 −17
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ 
  become: yes

  vars:

    # Set default if not provided in inventory

    _env_name: "{{ env_name | default('p368') }}"

    _env: "{{ env | default('p368') }}"

    _deploy_dir: "{{ deploy_dir | default('/opt/csst-airflow') }}"

    local_project_dir: ".."
@@ -56,45 +56,104 @@ 
  hosts: master

  become: yes

  vars:

    _env_name: "{{ env_name | default('p368') }}"

    _env: "{{ env | default('p368') }}"

    _deploy_dir: "{{ deploy_dir | default('/opt/csst-airflow') }}"

  tasks:

    - name: Ensure envs directory exists

    - name: Ensure deploy_configs directory exists

      file:

        path: "{{ _deploy_dir }}/envs"

        path: "{{ _deploy_dir }}/deploy_configs/{{ _env }}"

        state: directory



    - name: Render .env file from template

      template:

        src: "{{ playbook_dir }}/roles/common/templates/env.j2"

        dest: "{{ _deploy_dir }}/envs/{{ _env_name }}.env"

    - name: Copy airflow.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/airflow.env"

        dest: "{{ _deploy_dir }}/deploy_configs/airflow.env"



    - name: Copy variables.toml to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/variables.toml"

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/variables.toml"

      ignore_errors: yes



    - name: Copy connections.json to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/connections.json"

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/connections.json"

      ignore_errors: yes



    - name: Start Master services (postgres, redis, elasticsearch, airflow core, api, etc.)

      shell: |

        {{ docker_compose_cmd }} --env-file envs/{{ _env_name }}.env --profile master up -d

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export IMAGE_TAG="{{ image_tag | default('latest') }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ') }}"

        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env pull || true

        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env --profile master up -d --remove-orphans

      args:

        chdir: "{{ _deploy_dir }}"



    - name: Wait for Airflow Webserver to be ready before importing vars

      wait_for:

        port: 38080

        delay: 10

        timeout: 60

      ignore_errors: yes



    - name: Import Airflow Variables from variables.toml (if exists)

      shell: |

        python3 -c 'import sys, json; import tomllib; json.dump(tomllib.load(sys.stdin.buffer), sys.stdout)' < deploy_configs/{{ _env }}/variables.toml > deploy_configs/{{ _env }}/variables_parsed.json

        docker exec csst-airflow-airflow-apiserver-1 airflow variables import deploy_configs/{{ _env }}/variables_parsed.json

        rm deploy_configs/{{ _env }}/variables_parsed.json

      args:

        chdir: "{{ _deploy_dir }}"

      ignore_errors: yes



    - name: Import Airflow Connections from connections.json (if exists)

      shell: |

        docker exec csst-airflow-airflow-apiserver-1 airflow connections import deploy_configs/{{ _env }}/connections.json

      args:

        chdir: "{{ _deploy_dir }}"

      ignore_errors: yes



- name: Deploy Worker Node Services

  hosts: worker

  become: yes

  vars:

    _env_name: "{{ env_name | default('p368') }}"

    _env: "{{ env | default('p368') }}"

    _deploy_dir: "{{ deploy_dir | default('/opt/csst-airflow') }}"

  tasks:

    - name: Ensure envs directory exists

    - name: Ensure deploy_configs directory exists

      file:

        path: "{{ _deploy_dir }}/envs"

        path: "{{ _deploy_dir }}/deploy_configs/{{ _env }}"

        state: directory



    - name: Render .env file from template

      template:

        src: "{{ playbook_dir }}/roles/common/templates/env.j2"

        dest: "{{ _deploy_dir }}/envs/{{ _env_name }}.env"

    - name: Copy airflow.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/airflow.env"

        dest: "{{ _deploy_dir }}/deploy_configs/airflow.env"



    - name: Start Worker services (celery-worker, filebeat, monitoring)

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export IMAGE_TAG="{{ image_tag | default('latest') }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ') }}"

        export AIRFLOW__CELERY__WORKER_CONCURRENCY={{ worker_concurrency | default(16) }}

        {{ docker_compose_cmd }} --env-file envs/{{ _env_name }}.env --profile worker up -d

        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env pull || true

        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env --profile worker up -d --remove-orphans

      args:

        chdir: "{{ _deploy_dir }}"



- name: Post-Deploy Master Node Services (Ensure worker discovery)

  hosts: master

  become: yes

  vars:

    _env: "{{ env | default('p368') }}"

    _deploy_dir: "{{ deploy_dir | default('/opt/csst-airflow') }}"

  tasks:

    - name: Restart Flower and API Gateway to ensure they detect new workers

      shell: |

        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env restart flower api-gateway task-portal

      args:

        chdir: "{{ _deploy_dir }}"

      ignore_errors: yes