Commit 62a8cac4 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

fulfil ENVVARS

parent 292ea372

docker-celery/docker-compose.yaml→docker-celery-3.0.5/docker-compose.original.yaml

+3 −8
Original line number Diff line number Diff line 
# https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user

# curl -LfO 'https://airflow.apache.org/docs/apache-airflow/3.0.4/docker-compose.yaml'

#

# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements.  See the NOTICE file

# distributed with this work for additional information
@@ -108,10 +105,8 @@ services: 
    # Redis is limited to 7.2-bookworm due to licencing change

    # https://redis.io/blog/redis-adopts-dual-source-available-licensing/

    image: redis:7.2-bookworm

#    expose:

#      - 6379

    ports:

      - "6379:6379"

    expose:

      - 6379

    healthcheck:

      test: ["CMD", "redis-cli", "ping"]

      interval: 10s
@@ -124,7 +119,7 @@ services: 
    <<: *airflow-common

    command: api-server

    ports:

      - "38080:8080"

      - "8080:8080"

    healthcheck:

      test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/version"]

      interval: 30s

docker-celery-3.0.5/docker-compose.worker.yaml

deleted100644 → 0
+0 −65
Original line number Diff line number Diff line 
---

x-airflow-common:

  &airflow-common

  # In order to add custom dependencies or upgrade provider distributions you can use your extended image.

  # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml

  # and uncomment the "build" line below, Then run `docker-compose build` to build the images.

  image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:3.0.5}

  # build: .

  environment:

    &airflow-common-env

    AIRFLOW__CORE__EXECUTOR: CeleryExecutor

    AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager

    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@${MASTER_IP}:35432/airflow

    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@${MASTER_IP}:35432/airflow

    AIRFLOW__CELERY__BROKER_URL: redis://:@${MASTER_IP}:36379/0

    AIRFLOW__CORE__FERNET_KEY: ''

    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'

    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'

    AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/'

    # yamllint disable rule:line-length

    # Use simple http server on scheduler for health checks

    # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server

    # yamllint enable rule:line-length

    AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'

    # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks

    # for other purpose (development, test and especially production usage) build/extend Airflow image.

    _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}

    # The following line can be used to set a custom config file, stored in the local config folder

    AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'

    # REST API

    AIRFLOW__API_AUTH__JWT_SECRET: "${JWT_SECRET}"  # 从 .env 注入

    AIRFLOW__API_AUTH__JWT_ALGORITHM: "HS512"

    AIRFLOW__API_AUTH__JWT_EXPIRATION_TIME: 86400000 # 1000 days

  volumes:

    - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags

    - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs

    - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config

    - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins

  user: "${AIRFLOW_UID:-50000}:0"



services:

  airflow-worker:

    <<: *airflow-common

    command: celery worker

    healthcheck:

      # yamllint disable rule:line-length

      test:

        - "CMD-SHELL"

        - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'

      interval: 30s

      timeout: 10s

      retries: 5

      start_period: 30s

    environment:

      <<: *airflow-common-env

      # Required to handle warm shutdown of the celery workers properly

      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation

      DUMB_INIT_SETSID: "0"

    restart: always

#    depends_on:

#      <<: *airflow-common-depends-on

#      airflow-apiserver:

#        condition: service_healthy

#      airflow-init:

#        condition: service_completed_successfully
 
 No newline at end of file

docker-celery-3.0.5/docker-compose.yaml

+52 −19
Original line number Diff line number Diff line
@@ -81,10 +81,41 @@ x-airflow-common: 
    _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}

    # The following line can be used to set a custom config file, stored in the local config folder

    AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'

    # REST API

    # ===========================

    # Customize airflow.cfg below

    # ===========================

    # DEFAULT_TIMEZONE

    AIRFLOW__CORE__DEFAULT_TIMEZONE: "Asia/Shanghai" # "utc"

    AIRFLOW__CORE__PARALLELISM: 1024  # Max task instances per scheduler

    AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG: 1024 # Max active tasks per DAG

    AIRFLOW__CORE__MAX_ACTIVE_RUNS_PER_DAG: 256 # Max active DAG runs

    AIRFLOW__CORE__TEST_CONNECTION: 'Enabled'

    AIRFLOW__LOGGING__LOGGING_LEVEL: "INFO"

    # API

    AIRFLOW__API__HOST: "${MASTER_IP}"

    AIRFLOW__API__PORT: 38080

    AIRFLOW__API__WORKERS: 4

    # API AUTH

    AIRFLOW__API_AUTH__JWT_SECRET: "${JWT_SECRET}"  # 从 .env 注入

    AIRFLOW__API_AUTH__JWT_ALGORITHM: "HS512"

    AIRFLOW__API_AUTH__JWT_EXPIRATION_TIME: 86400000 # 1000 days

    # Operators

    AIRFLOW__OPERATORS__DEFAULT_CPUS: 1

    AIRFLOW__OPERATORS__DEFAULT_RAM: 1024

    AIRFLOW__OPERATORS__DEFAULT_DISK: 1024

    AIRFLOW__OPERATORS__DEFAULT_GPUS: 0

    AIRFLOW__OPERATORS__DEFAULT_QUEUE: "default"

    # SMTP?

    # Scheduler

    AIRFLOW__SCHEDULER__CATCHUP_BY_DEFAULT: 'false'

    AIRFLOW__SCHEDULER__IGNORE_FIRST_DEPENDS_ON_PAST_BY_DEFAULT: 'true'

    # CELERY FLOWER

    AIRFLOW__CELERY__FLOWER_BASIC_AUTH: "${_AIRFLOW_WWW_USER_USERNAME}:${_AIRFLOW_WWW_USER_PASSWORD}"

    AIRFLOW__CELERY__FLOWER_HOST: "${MASTER_IP}"

    AIRFLOW__CELERY__FLOWER_PORT: 35555

    AIRFLOW__CELERY__WORKER_CONCURRENCY: 64

    AIRFLOW__CELERY__WORKER_AUTOSCALE: "1,64"

    AIRFLOW__CELERY__WORKER_PREFETCH_MULTIPLIER: 1

  volumes:

    - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags

    - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs
@@ -180,24 +211,26 @@ services: 
      airflow-init:

        condition: service_completed_successfully



#  airflow-worker:

#    <<: *airflow-common

#    command: celery worker

#    healthcheck:

#      # yamllint disable rule:line-length

#      test:

#        - "CMD-SHELL"

#        - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'

#      interval: 30s

#      timeout: 10s

#      retries: 5

#      start_period: 30s

#    environment:

#      <<: *airflow-common-env

#      # Required to handle warm shutdown of the celery workers properly

#      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation

#      DUMB_INIT_SETSID: "0"

#    restart: always

  airflow-worker:

    <<: *airflow-common

    command: celery worker

    profiles:

      - worker

    healthcheck:

      # yamllint disable rule:line-length

      test:

        - "CMD-SHELL"

        - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'

      interval: 30s

      timeout: 10s

      retries: 5

      start_period: 30s

    environment:

      <<: *airflow-common-env

      # Required to handle warm shutdown of the celery workers properly

      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation

      DUMB_INIT_SETSID: "0"

    restart: always

#    depends_on:

#      <<: *airflow-common-depends-on

#      airflow-apiserver:

docker-celery-3.0.5/install.sh

0 → 100644
+3 −0
Original line number Diff line number Diff line 
docker compose -f docker-compose.yaml --env-file .env.p368 --profile flower up --force-recreate



docker compose -f docker-compose.yaml --env-file .env.p368 up worker

docker-celery/.env

deleted100644 → 0
+0 −10
Original line number Diff line number Diff line 
#AIRFLOW_IMAGE_NAME=apache/airflow:3.0.4

AIRFLOW_IMAGE_NAME=csst-airflow

AIRFLOW_UID=1000

AIRFLOW_PROJ_DIR=.

_AIRFLOW_WWW_USER_USERNAME=airflow

_AIRFLOW_WWW_USER_PASSWORD=airflow



JWT_SECRET=eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ

CSST_DFS_GATEWAY=10.80.1.22:28000

CSST_DFS_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjQ4ODU0NTA2NjQsInN1YiI6InN5c3RlbSJ9.POsuUABytu8-WMtZiYehiYEa5BnlgqNTXT6X3OTyix0