Commit 8313d0f0 authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

add 3.0.5

parent 49fe4c82

docker-celery-3.0.5/.env.p368

0 → 100644
+9 −0
Original line number Diff line number Diff line 
#AIRFLOW_IMAGE_NAME=apache/airflow:3.0.4

AIRFLOW_IMAGE_NAME=csst-airflow

AIRFLOW_UID=1000

AIRFLOW_PROJ_DIR=.

_AIRFLOW_WWW_USER_USERNAME=airflow

_AIRFLOW_WWW_USER_PASSWORD=airflow



JWT_SECRET=eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJhaXJmbG93IiwiaWF0IjoxNzU1NDQ3NjMxLCJleHAiOjE3ODY5ODM2MzF9.CbxQayEt370iHhvYXhHMESsk1IWmi8QLMOcctIlbicXhybNtBbfboDUdUDMClwvkSuTjC0AOeSHL23pGyy0plQ

MASTER_IP=10.73.0.11
 
 No newline at end of file

docker-celery-3.0.5/Dockerfile

0 → 100644
+10 −0
Original line number Diff line number Diff line 
FROM apache/airflow:3.0.5



COPY requirements.txt .



RUN pip install --upgrade pip

# switch to tsinghua pip/conda source

RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple/ \

    && pip config set global.extra-index-url "https://mirrors.bfsu.edu.cn/pypi/web/simple"

# install requirements

RUN pip install --no-cache-dir -r requirements.txt

docker-celery-3.0.5/README.md

0 → 100644
+9 −0
Original line number Diff line number Diff line 
#



## 



```shell







```
 
 No newline at end of file

docker-celery-3.0.5/docker-compose.worker.yaml

0 → 100644
+65 −0
Original line number Diff line number Diff line 
---

x-airflow-common:

  &airflow-common

  # In order to add custom dependencies or upgrade provider distributions you can use your extended image.

  # Comment the image line, place your Dockerfile in the directory where you placed the docker-compose.yaml

  # and uncomment the "build" line below, Then run `docker-compose build` to build the images.

  image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:3.0.5}

  # build: .

  environment:

    &airflow-common-env

    AIRFLOW__CORE__EXECUTOR: CeleryExecutor

    AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager

    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@${MASTER_IP}:35432/airflow

    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@${MASTER_IP}:35432/airflow

    AIRFLOW__CELERY__BROKER_URL: redis://:@${MASTER_IP}:36379/0

    AIRFLOW__CORE__FERNET_KEY: ''

    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'

    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'

    AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/'

    # yamllint disable rule:line-length

    # Use simple http server on scheduler for health checks

    # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server

    # yamllint enable rule:line-length

    AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'

    # WARNING: Use _PIP_ADDITIONAL_REQUIREMENTS option ONLY for a quick checks

    # for other purpose (development, test and especially production usage) build/extend Airflow image.

    _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}

    # The following line can be used to set a custom config file, stored in the local config folder

    AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'

    # REST API

    AIRFLOW__API_AUTH__JWT_SECRET: "${JWT_SECRET}"  # 从 .env 注入

    AIRFLOW__API_AUTH__JWT_ALGORITHM: "HS512"

    AIRFLOW__API_AUTH__JWT_EXPIRATION_TIME: 86400000 # 1000 days

  volumes:

    - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags

    - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs

    - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config

    - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins

  user: "${AIRFLOW_UID:-50000}:0"



services:

  airflow-worker:

    <<: *airflow-common

    command: celery worker

    healthcheck:

      # yamllint disable rule:line-length

      test:

        - "CMD-SHELL"

        - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'

      interval: 30s

      timeout: 10s

      retries: 5

      start_period: 30s

    environment:

      <<: *airflow-common-env

      # Required to handle warm shutdown of the celery workers properly

      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation

      DUMB_INIT_SETSID: "0"

    restart: always

#    depends_on:

#      <<: *airflow-common-depends-on

#      airflow-apiserver:

#        condition: service_healthy

#      airflow-init:

#        condition: service_completed_successfully
 
 No newline at end of file

docker-celery-3.0.5/docker-compose.yaml

+23 −7
Original line number Diff line number Diff line 
# curl -LfO 'https://airflow.apache.org/docs/apache-airflow/3.0.4/docker-compose.yaml'

# changes:

# - JWT secret

# - persistent PG data volume

# - ports

#   - airflow-apiserver: "38080:8080"

#   - redis: "36379:6379"

#   - flower: "35555:5555"

#

# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements.  See the NOTICE file
@@ -59,10 +66,10 @@ x-airflow-common: 
    AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager

    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow

    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow

    AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0

    AIRFLOW__CELERY__BROKER_URL: redis://:@redis:36379/0

    AIRFLOW__CORE__FERNET_KEY: ''

    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'

    AIRFLOW__CORE__LOAD_EXAMPLES: 'true'

    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'

    AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/'

    # yamllint disable rule:line-length

    # Use simple http server on scheduler for health checks
@@ -74,6 +81,10 @@ x-airflow-common: 
    _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}

    # The following line can be used to set a custom config file, stored in the local config folder

    AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'

    # REST API

    AIRFLOW__API_AUTH__JWT_SECRET: "${JWT_SECRET}"  # 从 .env 注入

    AIRFLOW__API_AUTH__JWT_ALGORITHM: "HS512"

    AIRFLOW__API_AUTH__JWT_EXPIRATION_TIME: 86400000 # 1000 days

  volumes:

    - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags

    - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs
@@ -95,20 +106,25 @@ services: 
      POSTGRES_PASSWORD: airflow

      POSTGRES_DB: airflow

    volumes:

      - postgres-db-volume:/var/lib/postgresql/data

#      - postgres-db-volume:/var/lib/postgresql/data

      - ${AIRFLOW_PROJ_DIR:-.}/pg_data:/var/lib/postgresql/data

    healthcheck:

      test: ["CMD", "pg_isready", "-U", "airflow"]

      interval: 10s

      retries: 5

      start_period: 5s

    restart: always

    ports:

      - "35432:5432"



  redis:

    # Redis is limited to 7.2-bookworm due to licencing change

    # https://redis.io/blog/redis-adopts-dual-source-available-licensing/

    image: redis:7.2-bookworm

    expose:

      - 6379

#    expose:

#      - 6379

    ports:

      - "36379:6379"

    healthcheck:

      test: ["CMD", "redis-cli", "ping"]

      interval: 10s
@@ -121,7 +137,7 @@ services: 
    <<: *airflow-common

    command: api-server

    ports:

      - "8080:8080"

      - "38080:8080"

    healthcheck:

      test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/version"]

      interval: 30s
@@ -320,7 +336,7 @@ services: 
    profiles:

      - flower

    ports:

      - "5555:5555"

      - "35555:5555"

    healthcheck:

      test: ["CMD", "curl", "--fail", "http://localhost:5555/"]

      interval: 30s