Commit 398cb773 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

chore: remove unused local deploy_configs, stick to p368

parent ef07ae87

docker-celery-3.0.5/Makefile

+66 −20
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ REQ_FILE := required-images.txt 
TARGET_AIRFLOW := $(HARBOR_PROJECT)csst-airflow:$(IMAGE_TAG)

TARGET_API_GATEWAY := $(HARBOR_PROJECT)csst-airflow-api-gateway:$(IMAGE_TAG)

TARGET_TASK_PORTAL := $(HARBOR_PROJECT)csst-airflow-task-portal:$(IMAGE_TAG)

TARGET_JS9 := $(HARBOR_PROJECT)csst-airflow-js9:$(IMAGE_TAG)



help:

	@echo "======================================================================"
@@ -22,11 +23,17 @@ help: 
	@echo "  make build       - Pull, retag third-party images, and build custom images"

	@echo "  make push	- Push all prepared images to the specified Harbor registry"

	@echo "  make clean       - Remove all custom built images (Airflow, API Gateway, Portal)"

	@echo "  make viewer-test - Build and run js9-service locally with ../data mounted"

	@echo "  make all	 - Execute clean, pull, build, and push sequentially"

	@echo "  make ansible-deploy - Deploy to environment (e.g. make ansible-deploy ENV=p368)"

	@echo "  make ansible-update - Update environment (e.g. make ansible-update ENV=p368)"

	@echo "  make ansible-down   - Tear down environment (e.g. make ansible-down ENV=p368)"

	@echo "  make ansible-check  - Run check playbook (e.g. make ansible-check ENV=p368)"

	@echo "  make ansible-sync-dags - Sync DAGs to environment (e.g. make ansible-sync-dags ENV=p368)"

	@echo ""

	@echo "Images to be processed:"

	@echo "  Third-party: postgres, redis, elasticsearch, kibana, filebeat, prometheus, grafana, cadvisor, node-exporter"

	@echo "  Custom:      csst-airflow, csst-airflow-api-gateway, csst-airflow-task-portal"

	@echo "  Custom:      csst-airflow, csst-airflow-api-gateway, csst-airflow-task-portal, csst-airflow-js9"

	@echo "======================================================================"



list-envs:
@@ -36,16 +43,16 @@ list-envs: 
build:

	@if [ ! -f $(REQ_FILE) ]; then echo "Error: $(REQ_FILE) not found"; exit 1; fi

	@total=$$(grep -v '^#' $(REQ_FILE) | grep -v '^$$' | wc -l); \

	echo "Pulling, retagging and building all $$(($$total+3)) images..."; \

	echo "Pulling, retagging and building all $$(($$total+4)) images..."; \

	count=1; \

	for img in $$(grep -v '^#' $(REQ_FILE) | grep -v '^$$'); do \

		basename="$${img##*/}"; \

		name="$${basename%%:*}"; \

		target="$(HARBOR_PROJECT)$${name}:$(IMAGE_TAG)"; \

		if docker image inspect $${img} > /dev/null 2>&1; then \

			echo "[$${count}/$$(($$total+3))] Processing $${img} -> $${target} (Skipping pull, already exists)..."; \

			echo "[$${count}/$$(($$total+4))] Processing $${img} -> $${target} (Skipping pull, already exists)..."; \

		else \

			echo "[$${count}/$$(($$total+3))] Processing $${img} -> $${target}..."; \

			echo "[$${count}/$$(($$total+4))] Processing $${img} -> $${target}..."; \

			(docker pull $${img} > /dev/null 2>&1 || true); \

		fi; \

		if [ "$${img}" != "$${target}" ]; then \
@@ -53,42 +60,81 @@ build: 
		fi; \

		count=$$((count+1)); \

	done; \

	echo "[$$(($$total+1))/$$(($$total+3))] Building Airflow Custom Image ($(TARGET_AIRFLOW))..."; \

	echo "[$$(($$total+1))/$$(($$total+4))] Building Airflow Custom Image ($(TARGET_AIRFLOW))..."; \

	docker build -q -t $(TARGET_AIRFLOW) .; \

	echo "[$$(($$total+2))/$$(($$total+3))] Building API Gateway Image ($(TARGET_API_GATEWAY))..."; \

	echo "[$$(($$total+2))/$$(($$total+4))] Building API Gateway Image ($(TARGET_API_GATEWAY))..."; \

	if [ -d "../external/csst-dfs-client" ]; then \

		cp -r ../external/csst-dfs-client ./api_gateway/; \

	fi; \

	docker build -q -t $(TARGET_API_GATEWAY) ./api_gateway; \

	echo "[$$(($$total+3))/$$(($$total+3))] Building Task Portal Image ($(TARGET_TASK_PORTAL)) with BUILD_NUMBER=$(BUILD_NUMBER)..."; \

	rm -rf ./api_gateway/csst-dfs-client; \

	echo "[$$(($$total+3))/$$(($$total+4))] Building Task Portal Image ($(TARGET_TASK_PORTAL)) with BUILD_NUMBER=$(BUILD_NUMBER)..."; \

	docker build -q --build-arg VITE_BUILD_NUMBER=$(BUILD_NUMBER) -t $(TARGET_TASK_PORTAL) ./frontend_vue; \

	echo "All $$(($$total+3)) images prepared successfully."

	echo "[$$(($$total+4))/$$(($$total+4))] Building JS9 Image ($(TARGET_JS9))..."; \

	docker build -q -t $(TARGET_JS9) -f ./js9_service/Dockerfile .; \

	echo "All $$(($$total+4)) images prepared successfully."



push:

	@if [ -z "$(HARBOR_PROJECT)" ]; then echo "Error: HARBOR_PROJECT is empty. Please specify a registry prefix to push. Example: make push HARBOR_PROJECT=csu-harbor.csst.nao:10443/csst/"; exit 1; fi

	@total=$$(grep -v '^#' $(REQ_FILE) | grep -v '^$$' | wc -l); \

	echo "Pushing all $$(($$total+3)) images to $(HARBOR_PROJECT)..."; \

	echo "Pushing all $$(($$total+4)) images to $(HARBOR_PROJECT)..."; \

	count=1; \

	for img in $$(grep -v '^#' $(REQ_FILE) | grep -v '^$$'); do \

		basename="$${img##*/}"; \

		name="$${basename%%:*}"; \

		target="$(HARBOR_PROJECT)$${name}:$(IMAGE_TAG)"; \

		echo "[$${count}/$$(($$total+3))] Pushing $${target}..."; \

		echo "[$${count}/$$(($$total+4))] Pushing $${target}..."; \

		docker push $${target}; \

		count=$$((count+1)); \

	done; \

	echo "[$$(($$total+1))/$$(($$total+3))] Pushing $(TARGET_AIRFLOW)..." && docker push $(TARGET_AIRFLOW); \

	echo "[$$(($$total+2))/$$(($$total+3))] Pushing $(TARGET_API_GATEWAY)..." && docker push $(TARGET_API_GATEWAY); \

	echo "[$$(($$total+3))/$$(($$total+3))] Pushing $(TARGET_TASK_PORTAL)..." && docker push $(TARGET_TASK_PORTAL); \

	echo "All $$(($$total+3)) images pushed successfully to $(HARBOR_PROJECT) with tag $(IMAGE_TAG)."

	echo "[$$(($$total+1))/$$(($$total+4))] Pushing $(TARGET_AIRFLOW)..." && docker push $(TARGET_AIRFLOW); \

	echo "[$$(($$total+2))/$$(($$total+4))] Pushing $(TARGET_API_GATEWAY)..." && docker push $(TARGET_API_GATEWAY); \

	echo "[$$(($$total+3))/$$(($$total+4))] Pushing $(TARGET_TASK_PORTAL)..." && docker push $(TARGET_TASK_PORTAL); \

	echo "[$$(($$total+4))/$$(($$total+4))] Pushing $(TARGET_JS9)..." && docker push $(TARGET_JS9); \

	echo "All $$(($$total+4)) images pushed successfully to $(HARBOR_PROJECT) with tag $(IMAGE_TAG)."



all: clean pull build push



clean:

	@echo "Cleaning up custom-built images..."

	@docker images | grep -E 'csst-airflow|api-gateway|task-portal' | awk '{print $$3}' | xargs -r docker rmi -f || true

	@docker images | grep -E 'csst-airflow|api-gateway|task-portal|js9' | awk '{print $$3}' | xargs -r docker rmi -f || true

	@echo "Clean completed."



viewer-test:

	@ENV=local HOST_DATA_DIR=../data JS9_SERVICE_PORT=38611 docker compose --env-file deploy_configs/airflow.env up -d js9-service --build

	@echo "JS9 test service is available at http://127.0.0.1:38611"



viewer-test-down:

	@ENV=local HOST_DATA_DIR=../data JS9_SERVICE_PORT=38611 docker compose --env-file deploy_configs/airflow.env rm -s -f js9-service



update-code:

	git pull



ansible-deploy:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-deploy ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/deploy.yml -e env=$(ENV)



ansible-update:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-update ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/update.yml -e env=$(ENV)



ansible-down:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-down ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/down.yml -e env=$(ENV)



ansible-check:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-check ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/check.yml -e env=$(ENV)



ansible-sync-dags:

	@if [ -z "$(ENV)" ]; then echo "Error: ENV is not set. Usage: make ansible-sync-dags ENV=p368"; exit 1; fi

	@if [ ! -f "deploy_configs/$(ENV)/inventory.ini" ]; then echo "Error: deploy_configs/$(ENV)/inventory.ini not found"; exit 1; fi

	ansible-playbook -i deploy_configs/$(ENV)/inventory.ini ansible/sync_dags.yml -e env=$(ENV)



pull:

	@total=$$(grep -v '^#' $(REQ_FILE) | grep -v '^$$' | wc -l); \

	echo "Required official images to pull ($$total in total):"; \

docker-celery-3.0.5/ansible/clean.yml

0 → 100644
+12 −0
Original line number Diff line number Diff line 
---

- name: Clean unused Docker resources on Master and Worker nodes

  hosts: all

  become: yes

  tasks:

    - name: Clean up dangling docker images (unused/untagged images)

      command: docker image prune -f

      ignore_errors: yes



    - name: Clean up unused build cache (optional)

      command: docker builder prune -f

      ignore_errors: yes

docker-celery-3.0.5/ansible/deploy.yml

+24 −0
Original line number Diff line number Diff line
@@ -69,6 +69,18 @@ 
        src: "{{ playbook_dir }}/../deploy_configs/airflow.env"

        dest: "{{ _deploy_dir }}/deploy_configs/airflow.env"



    - name: Copy gateway.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/gateway.env"

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/gateway.env"

      ignore_errors: yes



    - name: Copy js9.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/js9.env"

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/js9.env"

      ignore_errors: yes



    - name: Copy variables.toml to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/variables.toml"
@@ -132,6 +144,18 @@ 
        src: "{{ playbook_dir }}/../deploy_configs/airflow.env"

        dest: "{{ _deploy_dir }}/deploy_configs/airflow.env"



    - name: Copy gateway.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/gateway.env"

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/gateway.env"

      ignore_errors: yes



    - name: Copy js9.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/js9.env"

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/js9.env"

      ignore_errors: yes



    - name: Start Worker services (celery-worker, filebeat, monitoring)

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

docker-celery-3.0.5/ansible/update.yml

+43 −0
Original line number Diff line number Diff line
@@ -50,6 +50,18 @@ 
        src: "{{ playbook_dir }}/../deploy_configs/airflow.env"

        dest: "{{ _deploy_dir }}/deploy_configs/airflow.env"



    - name: Copy gateway.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/gateway.env"

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/gateway.env"

      ignore_errors: yes



    - name: Copy js9.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/js9.env"

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/js9.env"

      ignore_errors: yes



    - name: Copy variables.toml to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/variables.toml"
@@ -62,12 +74,23 @@ 
        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/connections.json"

      ignore_errors: yes



    - name: Clean up dangling docker images before update

      command: docker image prune -f

      ignore_errors: yes



    - name: Restart Master services (postgres, redis, elasticsearch, airflow core, api, etc.)

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export IMAGE_TAG="{{ image_tag | default('latest') }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ') }}"

        

        # Clean up target images to ensure we pull the fresh ones from registry

        docker rmi -f {{ harbor_project | default('') }}csst-airflow-api-gateway:{{ image_tag | default('latest') }} || true

        docker rmi -f {{ harbor_project | default('') }}csst-airflow-task-portal:{{ image_tag | default('latest') }} || true

        docker rmi -f {{ harbor_project | default('') }}csst-airflow-js9:{{ image_tag | default('latest') }} || true

        docker rmi -f {{ harbor_project | default('') }}csst-airflow:{{ image_tag | default('latest') }} || true



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env pull || true



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env --profile master up -d --remove-orphans
@@ -114,6 +137,22 @@ 
        src: "{{ playbook_dir }}/../deploy_configs/airflow.env"

        dest: "{{ _deploy_dir }}/deploy_configs/airflow.env"



    - name: Copy gateway.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/gateway.env"

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/gateway.env"

      ignore_errors: yes



    - name: Copy js9.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/js9.env"

        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/js9.env"

      ignore_errors: yes



    - name: Clean up dangling docker images before update

      command: docker image prune -f

      ignore_errors: yes



    - name: Restart Worker services (celery-worker, filebeat, monitoring)

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"
@@ -121,6 +160,10 @@ 
        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ') }}"

        export AIRFLOW__CELERY__WORKER_CONCURRENCY={{ worker_concurrency | default(16) }}

        

        # Clean up target images to ensure we pull the fresh ones from registry

        docker rmi -f {{ harbor_project | default('') }}csst-airflow:{{ image_tag | default('latest') }} || true



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env pull || true



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env --profile worker up -d --remove-orphans

docker-celery-3.0.5/api_gateway/Dockerfile

+5 −1
Original line number Diff line number Diff line
@@ -2,7 +2,11 @@ FROM python:3.11-slim 


WORKDIR /app

COPY requirements.txt .

RUN pip install --no-cache-dir -r requirements.txt



# Install dependencies, including local csst-dfs-client if it exists

COPY csst-dfs-clien[t] /app/csst-dfs-client/

RUN pip install --no-cache-dir -r requirements.txt && \

    if [ -f "/app/csst-dfs-client/setup.py" ] || [ -f "/app/csst-dfs-client/pyproject.toml" ]; then pip install --no-cache-dir /app/csst-dfs-client/; fi



COPY . .