Commit 474d5c96 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

docs: 更新文档和配置中的端口与部署信息

parent 609407de

README.md

+3 −0
Original line number Diff line number Diff line
@@ -182,8 +182,11 @@ CCDS_SERVER_URL = "http://10.73.0.27:29000" 
| Elasticsearch | `39200` | `http://<MASTER_IP>:39200` |

| PostgreSQL | `35432` | `postgresql://<MASTER_IP>:35432` |

| Redis | `36379` | `redis://<MASTER_IP>:36379` |

| cAdvisor | `38081` | `http://<MASTER_IP>:38081` |



Worker 节点通常不暴露核心业务端口，仅暴露监控相关端口供 Master 拉取。

- cAdvisor：`38081`

- Node Exporter：`9100`（注意：使用 `network_mode=host`，因此暴露的是宿主机默认端口）



## 对外接口

docker-celery-3.0.5/ansible/update.yml

+0 −4
Original line number Diff line number Diff line
@@ -141,7 +141,6 @@ 
        fi

        

        if [ "$REGISTRY_PULL" = "true" ]; then

          {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master down --remove-orphans || true

          {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master pull

        fi
@@ -257,10 +256,7 @@ 
          COMPOSE_FILES="$COMPOSE_FILES -f deploy_configs/{{ _env }}/docker-compose.extra-hosts.override.yml"

        fi



        {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master down --remove-orphans || true

        

        if [ "$REGISTRY_PULL" = "true" ]; then

          {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile master --profile worker down --remove-orphans || true

          {{ docker_compose_cmd }} $COMPOSE_FILES --env-file deploy_configs/airflow.env --profile worker pull

        fi

docker-celery-3.0.5/api_gateway/main.py

+11 −2
Original line number Diff line number Diff line
@@ -71,7 +71,7 @@ AIRFLOW_HOST = os.getenv("AIRFLOW_HOST", "airflow-apiserver") 
AIRFLOW_PORT = os.getenv("AIRFLOW_PORT", "8080")

AIRFLOW_URL = f"http://{AIRFLOW_HOST}:{AIRFLOW_PORT}"

FLOWER_HOST = os.getenv("FLOWER_HOST", "flower")

FLOWER_PORT = os.getenv("FLOWER_PORT", "35555")

FLOWER_PORT = os.getenv("FLOWER_PORT", "5555")

FLOWER_URL = f"http://{FLOWER_HOST}:{FLOWER_PORT}"

AIRFLOW_USER = os.getenv("AIRFLOW_USER", "airflow")

AIRFLOW_PASS = os.getenv("AIRFLOW_PASS", "airflow")
@@ -293,9 +293,14 @@ def batch_submit_tasks( 
    db_tasks = []

    seen_task_ids = set()

    

    group_info = payload.dag_group_run

    group_info = dict(payload.dag_group_run or {})

    tasks = payload.dag_run_list



    group_run_id = str(group_info.get("dag_group_run") or "").strip()

    if not group_run_id:

        group_run_id = str(uuid.uuid4())

        group_info["dag_group_run"] = group_run_id

    

    for task in tasks:

        # Fallback to existing fields if new fields are used

        requested_dag_id = task.get("dag") or task.get("dag_id")
@@ -312,6 +317,10 @@ def batch_submit_tasks( 
            

        new_task_id = str(dag_run_id)



        for k, v in group_info.items():

            if task.get(k) in (None, ""):

                task[k] = v

        

        # Merge group info into task inputs if needed, though they already have it

        # But we ensure dag_id and dag_run_id exist for backend logic

        task["requested_dag_id"] = requested_dag_id

docker-celery-3.0.5/docker-compose.yaml

+12 −9
Original line number Diff line number Diff line
@@ -118,11 +118,12 @@ x-airflow-common: 
    # CELERY FLOWER

    AIRFLOW__CELERY__FLOWER_BASIC_AUTH: "${_AIRFLOW_WWW_USER_USERNAME}:${_AIRFLOW_WWW_USER_PASSWORD}"

    AIRFLOW__CELERY__FLOWER_HOST: "0.0.0.0"

    AIRFLOW__CELERY__FLOWER_PORT: 35555

    AIRFLOW__CELERY__FLOWER_PORT: 5555

    AIRFLOW__CELERY__WORKER_SEND_TASK_EVENTS: "true"

    AIRFLOW__CELERY__WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-8}

    # AIRFLOW__CELERY__WORKER_AUTOSCALE: "1,64" # Disabled to allow manual slot adjustment via UI/Flower

    AIRFLOW__CELERY__WORKER_PREFETCH_MULTIPLIER: 1

    AIRFLOW__CELERY_BROKER_TRANSPORT_OPTIONS__VISIBILITY_TIMEOUT: ${CELERY_VISIBILITY_TIMEOUT:-86400}

  volumes:

    - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags

    - ${AIRFLOW_PROJ_DIR:-.}/volumes/logs:/opt/airflow/logs
@@ -298,6 +299,7 @@ services: 
      # Required to handle warm shutdown of the celery workers properly

      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation

      DUMB_INIT_SETSID: "0"

    stop_grace_period: ${WORKER_STOP_GRACE_PERIOD:-30m}

    restart: always

    # depends_on:

    #   <<: *airflow-common-depends-on
@@ -437,13 +439,13 @@ services: 
  # See: https://docs.docker.com/compose/profiles/

  flower:

    <<: *airflow-common

    command: celery flower

    command: celery flower --address=0.0.0.0 --port=5555

    profiles:

      - master

    ports:

      - "35555:35555"

      - "35555:5555"

    healthcheck:

      test: ["CMD", "curl", "--fail", "http://localhost:35555/"]

      test: ["CMD", "curl", "--fail", "http://localhost:5555/"]

      interval: 30s

      timeout: 10s

      retries: 5
@@ -464,11 +466,12 @@ services: 
    env_file:

      - ${AIRFLOW_PROJ_DIR:-.}/deploy_configs/${ENV:-local}/gateway.env

    environment:

      # 使用 docker 网络内部的服务名和端口，或者通过 .env 指定外部 IP

      POSTGRES_HOST: ${POSTGRES_HOST:-postgres}

      POSTGRES_PORT: ${POSTGRES_PORT:-5432}

      AIRFLOW_HOST: ${AIRFLOW_HOST:-airflow-apiserver}

      AIRFLOW_PORT: ${AIRFLOW_PORT:-8080}

      POSTGRES_HOST: postgres

      POSTGRES_PORT: 5432

      AIRFLOW_HOST: airflow-apiserver

      AIRFLOW_PORT: 8080

      FLOWER_HOST: flower

      FLOWER_PORT: 5555

      POSTGRES_USER: ${POSTGRES_USER:-airflow}

      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-airflow}

      POSTGRES_DB: ${POSTGRES_DB:-airflow}