Commit 69b99603 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

chore: 移除外部子模块并更新部署配置

parent 954c7486

.gitignore

+1 −1
Original line number Diff line number Diff line
@@ -57,7 +57,7 @@ tests/ 
tmp/



# External files

external/

# external/

data/



# Generated during Ansible variable import

docker-celery-3.0.5/ansible/deploy.yml

+10 −0
Original line number Diff line number Diff line
@@ -55,6 +55,16 @@ 
          - "--exclude=ansible/*"

          - "--exclude=frontend_vue/node_modules"





    - name: Synchronize DAGs folder with deletion (avoid stale DAG variants)

      synchronize:

        src: "{{ playbook_dir }}/../dags/"

        dest: "{{ _deploy_dir }}/dags/"

        delete: yes

        recursive: yes

        rsync_opts:

          - "--exclude=__pycache__"



    - name: Ensure volume directories exist

      file:

        path: "{{ _deploy_dir }}/volumes/{{ item }}"

docker-celery-3.0.5/ansible/update.yml

+8 −0
Original line number Diff line number Diff line
@@ -196,6 +196,14 @@ 
        chdir: "{{ _deploy_dir }}"

      ignore_errors: yes



    - name: Deactivate legacy __low DAG ids (keep history, hide from UI)

      shell: |

        docker exec csst-airflow-postgres-1 psql -U airflow -d airflow -v ON_ERROR_STOP=1 \

          -c "UPDATE dag SET is_active = FALSE WHERE dag_id LIKE '%__low';"

      args:

        chdir: "{{ _deploy_dir }}"

      ignore_errors: yes



- name: Update Worker Node Services

  hosts: worker

  become: yes

docker-celery-3.0.5/api_gateway/main.py

+10 −2
Original line number Diff line number Diff line
@@ -741,8 +741,8 @@ def get_airflow_variables(db: Session = Depends(get_db), current_user: User = De 
def get_flower_workers(db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):

    """Fetch Celery workers information from Flower API and merge with Airflow slots"""

    try:

        FLOWER_AUTH_USER = os.getenv("_AIRFLOW_WWW_USER_USERNAME", AIRFLOW_USER)

        FLOWER_AUTH_PASS = os.getenv("_AIRFLOW_WWW_USER_PASSWORD", AIRFLOW_PASS)

        FLOWER_AUTH_USER = os.getenv("FLOWER_AUTH_USER") or os.getenv("_AIRFLOW_WWW_USER_USERNAME") or AIRFLOW_USER

        FLOWER_AUTH_PASS = os.getenv("FLOWER_AUTH_PASS") or os.getenv("_AIRFLOW_WWW_USER_PASSWORD") or AIRFLOW_PASS

        inventory_worker_hosts = load_worker_inventory_hosts()

        

        response = requests.get(
@@ -753,6 +753,9 @@ def get_flower_workers(db: Session = Depends(get_db), current_user: User = Depen 
        response.raise_for_status()

        workers_data = response.json()



        if not isinstance(workers_data, dict):

            raise HTTPException(status_code=502, detail="Invalid Flower workers response")

        

        # Query Airflow DB for occupied slots per hostname

        try:

            query = text("""
@@ -789,6 +792,11 @@ def get_flower_workers(db: Session = Depends(get_db), current_user: User = Depen 
    except requests.exceptions.RequestException as e:

        logger.error(f"Failed to fetch workers from Flower: {str(e)}")

        raise HTTPException(status_code=502, detail="Failed to communicate with Flower API")

    except HTTPException:

        raise

    except Exception as e:

        logger.error(f"Failed to fetch workers from Flower: {str(e)}")

        raise HTTPException(status_code=502, detail="Failed to fetch workers")



@app.get("/api/dfs/level0/count_by_dataset")

def get_dfs_level0_count_by_dataset(debug: bool = False, current_user: User = Depends(get_current_user)):

docker-celery-3.0.5/docker-compose.yaml

+2 −0
Original line number Diff line number Diff line
@@ -472,6 +472,8 @@ services: 
      AIRFLOW_PORT: 8080

      FLOWER_HOST: flower

      FLOWER_PORT: 5555

      FLOWER_AUTH_USER: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}

      FLOWER_AUTH_PASS: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}

      POSTGRES_USER: ${POSTGRES_USER:-airflow}

      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-airflow}

      POSTGRES_DB: ${POSTGRES_DB:-airflow}