Commit b3685f06 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

fix: 修复前端缓存、部署脚本和清理逻辑，优化Harbor组件交互

parent c56b38ed

docker-celery-3.0.5/Makefile

+8 −1
Original line number Diff line number Diff line
@@ -150,7 +150,14 @@ all: clean pull build push 


clean:

	@echo "Cleaning up custom-built images..."

	@docker images | grep -E 'csst-airflow|api-gateway|task-portal|js9' | awk '{print $$3}' | xargs -r docker rmi -f || true

	@set -e; \

	ids=$$(docker images --format '{{.Repository}}:{{.Tag}} {{.ID}}' | awk '$$1 ~ /(csst-airflow|api-gateway|task-portal|js9)/ {print $$2}' | sort -u); \

	for id in $$ids; do \

		if docker ps -q --filter ancestor=$$id | grep -q .; then \

			continue; \

		fi; \

		docker rmi -f $$id || true; \

	done

	@echo "Cleaning dangling docker images and build cache..."

	@docker image prune -f >/dev/null 2>&1 || true

	@docker builder prune -f >/dev/null 2>&1 || true

docker-celery-3.0.5/ansible/deploy.yml

+31 −31
Original line number Diff line number Diff line
@@ -73,8 +73,6 @@ 
  vars:

    _env: "{{ env | default('p368') }}"

    _deploy_dir: "{{ deploy_dir | default('/opt/csst-airflow') }}"

    _build_task_portal: "{{ build_task_portal | default('false') }}"

    _build_api_gateway: "{{ build_api_gateway | default('false') }}"

    _extra_hosts_list: "{{ (extra_hosts | default('') ).split(',') | map('trim') | reject('equalto','') | list }}"

    _harbor_api_username: "{{ lookup('env', 'HARBOR_USERNAME') | default('', true) }}"

    _harbor_api_password: "{{ lookup('env', 'HARBOR_PASSWORD') | default('', true) }}"
@@ -134,34 +132,6 @@ 
        - _harbor_api_password | length > 0

      no_log: true



    - name: Build task-portal image (optional)

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export BUILD_NUMBER="{{ _build_number }}"

        export BUILD_TIME="{{ build_time | default(ansible_facts['date_time']['iso8601']) }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('') }}"

        export ENV="{{ _env }}"



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env build task-portal

      args:

        chdir: "{{ _deploy_dir }}"

      when: _build_task_portal | bool



    - name: Build api-gateway image (optional)

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export BUILD_NUMBER="{{ _build_number }}"

        export BUILD_TIME="{{ build_time | default(ansible_facts['date_time']['iso8601']) }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('') }}"

        export ENV="{{ _env }}"



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env build api-gateway

      args:

        chdir: "{{ _deploy_dir }}"

      when: _build_api_gateway | bool



    - name: Start Master services (postgres, redis, elasticsearch, airflow core, api, etc.)

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"
@@ -174,6 +144,8 @@ 
        if [ -f deploy_configs/{{ _env }}/docker-compose.extra-hosts.override.yml ]; then

          COMPOSE_FILES="$COMPOSE_FILES -f deploy_configs/{{ _env }}/docker-compose.extra-hosts.override.yml"

        fi



        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master down --remove-orphans || true

        if [ -f deploy_configs/{{ _env }}/docker-compose.harbor-api-auth.override.yml ]; then

          COMPOSE_FILES="$COMPOSE_FILES -f deploy_configs/{{ _env }}/docker-compose.harbor-api-auth.override.yml"

        fi
@@ -193,6 +165,18 @@ 
        docker rmi -f {{ harbor_project | default('') }}csst-airflow:{{ _build_number }} || true

        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master pull

        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master up -d --remove-orphans

        ensure_tag() {

          svc="$1"

          cid=$({{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master ps -q "$svc" 2>/dev/null || true)

          if [ -n "$cid" ]; then

            img_ref=$(docker inspect "$cid" --format '{{.Config.Image}}')

            img_id=$(docker inspect "$cid" --format '{{.Image}}')

            docker image inspect "$img_ref" >/dev/null 2>&1 || docker image tag "$img_id" "$img_ref"

          fi

        }

        for svc in postgres redis elasticsearch kibana filebeat prometheus grafana cadvisor node-exporter airflow-apiserver airflow-scheduler airflow-dag-processor airflow-triggerer flower api-gateway task-portal; do

          ensure_tag "$svc"

        done

      args:

        chdir: "{{ _deploy_dir }}"
@@ -283,7 +267,23 @@ 
        docker rmi -f {{ harbor_project | default('') }}node-exporter:{{ _build_number }} || true

        docker rmi -f {{ harbor_project | default('') }}csst-airflow:{{ _build_number }} || true

        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker pull

        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker up -d --remove-orphans

        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker up -d --remove-orphans --force-recreate

        ensure_tag() {

          svc="$1"

          cid=$({{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker ps -q "$svc" 2>/dev/null || true)

          if [ -n "$cid" ]; then

            img_ref=$(docker inspect "$cid" --format '{{.Config.Image}}')

            img_id=$(docker inspect "$cid" --format '{{.Image}}')

            docker image inspect "$img_ref" >/dev/null 2>&1 || docker image tag "$img_id" "$img_ref"

          fi

        }

        for svc in airflow-worker filebeat cadvisor node-exporter; do

          ensure_tag "$svc"

        done

        master_left=$({{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master ps -q || true)

        if [ -n "$master_left" ]; then

          {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master down --remove-orphans || true

        fi

      args:

        chdir: "{{ _deploy_dir }}"

docker-celery-3.0.5/ansible/update.yml

+31 −31
Original line number Diff line number Diff line
@@ -54,8 +54,6 @@ 
  vars:

    _env: "{{ env | default('p368') }}"

    _deploy_dir: "{{ deploy_dir | default('/opt/csst-airflow') }}"

    _build_task_portal: "{{ build_task_portal | default('false') }}"

    _build_api_gateway: "{{ build_api_gateway | default('false') }}"

    _extra_hosts_list: "{{ (extra_hosts | default('') ).split(',') | map('trim') | reject('equalto','') | list }}"

    _harbor_api_username: "{{ lookup('env', 'HARBOR_USERNAME') | default('', true) }}"

    _harbor_api_password: "{{ lookup('env', 'HARBOR_PASSWORD') | default('', true) }}"
@@ -125,34 +123,6 @@ 
      command: docker image prune -f

      ignore_errors: yes



    - name: Build task-portal image (optional)

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export BUILD_NUMBER="{{ _build_number }}"

        export BUILD_TIME="{{ build_time | default(ansible_facts['date_time']['iso8601']) }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('') }}"

        export ENV="{{ _env }}"



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env build task-portal

      args:

        chdir: "{{ _deploy_dir }}"

      when: _build_task_portal | bool



    - name: Build api-gateway image (optional)

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"

        export BUILD_NUMBER="{{ _build_number }}"

        export BUILD_TIME="{{ build_time | default(ansible_facts['date_time']['iso8601']) }}"

        export MASTER_IP="{{ hostvars[groups['master'][0]]['ansible_host'] | default('127.0.0.1') }}"

        export JWT_SECRET="{{ jwt_secret | default('') }}"

        export ENV="{{ _env }}"



        {{ docker_compose_cmd }} --env-file deploy_configs/airflow.env build api-gateway

      args:

        chdir: "{{ _deploy_dir }}"

      when: _build_api_gateway | bool



    - name: Restart Master services (postgres, redis, elasticsearch, airflow core, api, etc.)

      shell: |

        export HARBOR_PROJECT="{{ harbor_project | default('') }}"
@@ -176,6 +146,18 @@ 
        fi



        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master up -d --remove-orphans --force-recreate

        ensure_tag() {

          svc="$1"

          cid=$({{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master ps -q "$svc" 2>/dev/null || true)

          if [ -n "$cid" ]; then

            img_ref=$(docker inspect "$cid" --format '{{.Config.Image}}')

            img_id=$(docker inspect "$cid" --format '{{.Image}}')

            docker image inspect "$img_ref" >/dev/null 2>&1 || docker image tag "$img_id" "$img_ref"

          fi

        }

        for svc in postgres redis elasticsearch kibana filebeat prometheus grafana cadvisor node-exporter airflow-apiserver airflow-scheduler airflow-dag-processor airflow-triggerer flower api-gateway task-portal; do

          ensure_tag "$svc"

        done

        if [ "$REGISTRY_PULL" = "true" ]; then

          docker image prune -af || true

        fi
@@ -275,12 +257,30 @@ 
          COMPOSE_FILES="$COMPOSE_FILES -f deploy_configs/{{ _env }}/docker-compose.extra-hosts.override.yml"

        fi



        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master down --remove-orphans || true

        

        if [ "$REGISTRY_PULL" = "true" ]; then

          {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker down --remove-orphans || true

          {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master --profile worker down --remove-orphans || true

          {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker pull

        fi



        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker up -d --remove-orphans --force-recreate

        ensure_tag() {

          svc="$1"

          cid=$({{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker ps -q "$svc" 2>/dev/null || true)

          if [ -n "$cid" ]; then

            img_ref=$(docker inspect "$cid" --format '{{.Config.Image}}')

            img_id=$(docker inspect "$cid" --format '{{.Image}}')

            docker image inspect "$img_ref" >/dev/null 2>&1 || docker image tag "$img_id" "$img_ref"

          fi

        }

        for svc in airflow-worker filebeat cadvisor node-exporter; do

          ensure_tag "$svc"

        done

        master_left=$({{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master ps -q || true)

        if [ -n "$master_left" ]; then

          {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master down --remove-orphans || true

        fi

        if [ "$REGISTRY_PULL" = "true" ]; then

          docker image prune -af || true

        fi

docker-celery-3.0.5/frontend_vue/nginx.conf

+6 −0
Original line number Diff line number Diff line
@@ -2,6 +2,12 @@ server { 
    listen 38501;

    server_name _;



    location = / {

        add_header Cache-Control "no-cache, no-store, must-revalidate" always;

        root   /usr/share/nginx/html;

        try_files /index.html =404;

    }



    location = /index.html {

        add_header Cache-Control "no-cache, no-store, must-revalidate" always;

        root   /usr/share/nginx/html;

docker-celery-3.0.5/frontend_vue/src/App.vue

+11 −15
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@ 
  <LoginView v-if="!isAuthenticated" @login-success="handleLoginSuccess" />

  <div v-else :class="['h-screen flex bg-gray-50 overflow-hidden', fontSizeClass]">

    <!-- Sidebar -->

    <aside class="w-44 bg-[#1D1D1D] text-[#E0E0E0] flex-shrink-0 flex flex-col border-r border-[#2C2C2C] transition-all duration-300">

    <aside class="w-56 overflow-x-hidden bg-[#1D1D1D] text-[#E0E0E0] flex-shrink-0 flex flex-col border-r border-[#2C2C2C] transition-all duration-300">

      <!-- Logo Area -->

      <div class="h-14 flex items-center px-4 border-b border-[#2C2C2C]">

        <div class="text-lg font-semibold tracking-wide flex items-center">
@@ -17,7 +17,7 @@ 
        background-color="transparent"

        text-color="#A0A0A0"

        active-text-color="#FFFFFF"

        class="border-none flex-1 py-2 teamcity-vertical-menu overflow-y-auto"

        class="border-none flex-1 py-2 teamcity-vertical-menu overflow-y-auto overflow-x-hidden"

      >

        <div class="px-5 py-2 mt-2 text-[10px] font-bold text-[#6B7280] uppercase tracking-wider">Data</div>

        <el-menu-item index="plan" title="Plan">
@@ -58,16 +58,16 @@ 
          <el-icon :size="20" class="font-bold"><List /></el-icon>

          <span class="text-[15px] font-bold ml-2">DAGs</span>

        </el-menu-item>

        <el-menu-item index="workers">

          <el-icon :size="20" class="font-bold"><Cpu /></el-icon>

          <span class="text-[15px] font-bold ml-2">Workers</span>

        </el-menu-item>



        <div class="px-5 py-2 mt-4 text-[10px] font-bold text-[#6B7280] uppercase tracking-wider">Management</div>

        <el-menu-item index="env">

          <el-icon :size="20" class="font-bold"><Document /></el-icon>

          <span class="text-[15px] font-bold ml-2">Variables</span>

        </el-menu-item>

        <el-menu-item index="workers">

          <el-icon :size="20" class="font-bold"><Cpu /></el-icon>

          <span class="text-[15px] font-bold ml-2">Workers</span>

        </el-menu-item>

        <el-menu-item index="harbor-tags">

          <el-icon :size="20" class="font-bold"><PriceTag /></el-icon>

          <span class="text-[15px] font-bold ml-2">Harbor</span>
@@ -80,10 +80,6 @@ 
          <el-icon :size="20" class="font-bold"><DataLine /></el-icon>

          <span class="text-[15px] font-bold ml-2">Grafana</span>

        </el-menu-item>

        <el-menu-item index="test">

          <el-icon :size="20" class="font-bold"><Search /></el-icon>

          <span class="text-[15px] font-bold ml-2">Test</span>

        </el-menu-item>

        <el-menu-item index="admin" v-if="userRole === 'admin'">

          <el-icon :size="20" class="font-bold"><Setting /></el-icon>

          <span class="text-[15px] font-bold ml-2">Admin</span>
@@ -556,7 +552,7 @@ 
            <RawDataView v-else-if="activeTab === 'rawdata'" />

      </div>



      <div v-else-if="['grafana', 'dags', 'workers', 'env', 'harbor-tags', 'js9', 'admin', 'test'].includes(activeTab)" class="h-[calc(100vh-140px)] w-full rounded-lg overflow-hidden border border-[#E5E7EB] bg-white">

      <div v-else-if="['grafana', 'dags', 'workers', 'env', 'harbor-tags', 'js9', 'admin'].includes(activeTab)" class="h-[calc(100vh-140px)] w-full rounded-lg overflow-hidden border border-[#E5E7EB] bg-white">

        <iframe

              v-if="activeTab === 'grafana'"

              :src="`http://${currentHost}:33000/d/node-exporter/node-exporter-full?orgId=1&refresh=1m&kiosk`"
@@ -568,7 +564,6 @@ 
            <EnvView v-else-if="activeTab === 'env'" />

            <HarborTagsView v-else-if="activeTab === 'harbor-tags'" />

            <JS9View v-else-if="activeTab === 'js9'" />

            <TestView v-else-if="activeTab === 'test'" />

            <UsersView v-else-if="activeTab === 'admin' && userRole === 'admin'" />

      </div>

    </main>
@@ -591,7 +586,6 @@ import CCDSView from './components/CCDSView.vue' 
import JS9View from './components/JS9View.vue'

import SubmissionsView from './components/SubmissionsView.vue'

import HarborTagsView from './components/HarborTagsView.vue'

import TestView from './components/TestView.vue'

import axios from 'axios'

import { Monitor, DataLine, Search, Filter, Document, Clock, Close, Loading, Select, CircleClose, VideoPlay, Warning, Back, Refresh, Folder, ArrowRight, ArrowLeft, QuestionFilled, List, Cpu, Setting, SwitchButton, Position, Calendar, Notebook, Picture, Promotion, Box, Tickets, PriceTag } from '@element-plus/icons-vue'

import { ElMessage } from 'element-plus'
@@ -618,7 +612,7 @@ const API_BASE_URL = '/api/tasks' 


const getHashTab = () => {

  const hash = window.location.hash.replace('#', '')

  return ['submissions', 'tasks', 'dags', 'workers', 'env', 'harbor-tags', 'grafana', 'admin', 'plan', 'rawdata', 'products', 'catalogs', 'ccds', 'test'].includes(hash) ? hash : 'tasks'

  return ['submissions', 'tasks', 'dags', 'workers', 'env', 'harbor-tags', 'grafana', 'admin', 'plan', 'rawdata', 'products', 'catalogs', 'ccds'].includes(hash) ? hash : 'tasks'

}



// State
@@ -669,7 +663,6 @@ const activeTabTitle = computed(() => { 
    catalogs: 'External Catalogs',

    ccds: 'CCDS',

    js9: 'JS9',

    test: 'Test',

    admin: 'Admin'

  }

  return map[activeTab.value] || activeTab.value
@@ -1018,6 +1011,9 @@ onUnmounted(() => { 
  margin-bottom: 4px;

  border-left: 3px solid transparent;

}

.teamcity-vertical-menu {

  overflow-x: hidden;

}

.teamcity-collapse-inner .el-collapse-item__header {

  border-bottom: none;

  background-color: transparent;