Commit 587870c6 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

feat: 新增数据模块界面及多项功能改进

parent 398cb773

.gitignore

+1 −1
Original line number Diff line number Diff line
@@ -58,4 +58,4 @@ tmp/ 


# External files

external/

data/

docker-celery-3.0.5/Makefile

+15 −4
Original line number Diff line number Diff line
@@ -64,14 +64,25 @@ build: 
	docker build -q -t $(TARGET_AIRFLOW) .; \

	echo "[$$(($$total+2))/$$(($$total+4))] Building API Gateway Image ($(TARGET_API_GATEWAY))..."; \

		if [ -d "../external/csst-dfs-client" ]; then \

		cp -r ../external/csst-dfs-client ./api_gateway/; \

				mkdir -p ./api_gateway/csst-dfs-client; \

				rsync -a --delete --exclude ".git" --exclude "__pycache__" ../external/csst-dfs-client/ ./api_gateway/csst-dfs-client/; \

		fi; \

		if [ -d "../external/ccds" ]; then \

				mkdir -p ./api_gateway/ccds; \

				rsync -a --delete --exclude ".git" --exclude "__pycache__" ../external/ccds/ ./api_gateway/ccds/; \

		fi; \

	docker build -q -t $(TARGET_API_GATEWAY) ./api_gateway; \

	rm -rf ./api_gateway/csst-dfs-client; \

	rm -rf ./api_gateway/ccds; \

	echo "[$$(($$total+3))/$$(($$total+4))] Building Task Portal Image ($(TARGET_TASK_PORTAL)) with BUILD_NUMBER=$(BUILD_NUMBER)..."; \

	docker build -q --build-arg VITE_BUILD_NUMBER=$(BUILD_NUMBER) -t $(TARGET_TASK_PORTAL) ./frontend_vue; \

	docker build -q --build-arg VITE_BUILD_NUMBER=$(BUILD_NUMBER) --build-arg VITE_BUILD_TIME="$(shell date '+%Y-%m-%d %H:%M:%S')" -t $(TARGET_TASK_PORTAL) ./frontend_vue; \

	echo "[$$(($$total+4))/$$(($$total+4))] Building JS9 Image ($(TARGET_JS9))..."; \

		if [ -d "../external/js9" ]; then \

				mkdir -p ./external/js9; \

				rsync -a --delete --exclude ".git" --exclude "__pycache__" ../external/js9/ ./external/js9/; \

		fi; \

	docker build -q -t $(TARGET_JS9) -f ./js9_service/Dockerfile .; \

	rm -rf ./external/js9; \

	echo "All $$(($$total+4)) images prepared successfully."



push:

docker-celery-3.0.5/ansible/update.yml

+34 −10
Original line number Diff line number Diff line
@@ -56,6 +56,12 @@ 
        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/gateway.env"

      ignore_errors: yes



    - name: Remove legacy local deploy config on remote

      file:

        path: "{{ _deploy_dir }}/deploy_configs/local"

        state: absent

      ignore_errors: yes



    - name: Copy js9.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/js9.env"
@@ -84,14 +90,17 @@ 
        export IMAGE_TAG="{{ image_tag | default('latest') }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ') }}"

        export ENV="{{ _env }}"

        export REGISTRY_PULL="{{ registry_pull | default('false') }}"

        

        # Clean up target images to ensure we pull the fresh ones from registry

        if [ "$REGISTRY_PULL" = "true" ]; then

          docker rmi -f {{ harbor_project | default('') }}csst-airflow-api-gateway:{{ image_tag | default('latest') }} || true

          docker rmi -f {{ harbor_project | default('') }}csst-airflow-task-portal:{{ image_tag | default('latest') }} || true

          docker rmi -f {{ harbor_project | default('') }}csst-airflow-js9:{{ image_tag | default('latest') }} || true

          docker rmi -f {{ harbor_project | default('') }}csst-airflow:{{ image_tag | default('latest') }} || true



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env pull || true

          docker compose --env-file deploy_configs/airflow.env pull || true

        fi



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env --profile master up -d --remove-orphans

      args:
@@ -143,6 +152,12 @@ 
        dest: "{{ _deploy_dir }}/deploy_configs/{{ _env }}/gateway.env"

      ignore_errors: yes



    - name: Remove legacy local deploy config on remote

      file:

        path: "{{ _deploy_dir }}/deploy_configs/local"

        state: absent

      ignore_errors: yes



    - name: Copy js9.env to remote

      copy:

        src: "{{ playbook_dir }}/../deploy_configs/{{ _env }}/js9.env"
@@ -159,12 +174,15 @@ 
        export IMAGE_TAG="{{ image_tag | default('latest') }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ') }}"

        export ENV="{{ _env }}"

        export WORKER_LABEL="{{ ansible_host | default(inventory_hostname) }}"

        export AIRFLOW__CELERY__WORKER_CONCURRENCY={{ worker_concurrency | default(16) }}

        export REGISTRY_PULL="{{ registry_pull | default('false') }}"

        

        # Clean up target images to ensure we pull the fresh ones from registry

        if [ "$REGISTRY_PULL" = "true" ]; then

          docker rmi -f {{ harbor_project | default('') }}csst-airflow:{{ image_tag | default('latest') }} || true



          {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env pull || true

        fi



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env --profile worker up -d --remove-orphans

      args:
@@ -179,6 +197,12 @@ 
  tasks:

    - name: Restart Flower and API Gateway to ensure they detect new workers

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export IMAGE_TAG="{{ image_tag | default('latest') }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('') }}"

        export ENV="{{ _env }}"



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env restart flower api-gateway task-portal

      args:

        chdir: "{{ _deploy_dir }}"

docker-celery-3.0.5/api_gateway/.dockerignore

0 → 100644
+7 −0
Original line number Diff line number Diff line 
.git

__pycache__

*.pyc

csst-dfs-client/.git

csst-dfs-client/__pycache__

ccds/.git

ccds/__pycache__

docker-celery-3.0.5/api_gateway/Dockerfile

+4 −2
Original line number Diff line number Diff line
@@ -4,13 +4,15 @@ WORKDIR /app 
COPY requirements.txt .



# Install dependencies, including local csst-dfs-client if it exists

COPY csst-dfs-clien[t] /app/csst-dfs-client/

COPY csst-dfs-client /app/csst-dfs-client/

COPY ccds /app/ccds/

RUN pip install --no-cache-dir -r requirements.txt && \

    if [ -f "/app/csst-dfs-client/setup.py" ] || [ -f "/app/csst-dfs-client/pyproject.toml" ]; then pip install --no-cache-dir /app/csst-dfs-client/; fi

RUN if [ -f "/app/ccds/setup.py" ] || [ -f "/app/ccds/pyproject.toml" ]; then pip install --no-cache-dir /app/ccds/; fi



COPY . .



# 暴露网关端口

EXPOSE 38000



CMD ["python", "main.py"]
 
CMD ["python", "-u", "main.py"]