refactor(docker): simplify airflow deployment and update documentation

# Python

__pycache__/

*.pyc

# Docker

.env

volumes/

docker-compose.override.yaml

# Conda

miniconda3/

*.sh

# Logs

logs/

*.log

# IDE

.idea/

.vscode/

# Temporary files

*.tmp

*.temp

# OS

.DS_Store

Thumbs.db

# Build artifacts

build/

dist/

*.egg-info/

# Environment variables

.env.local

.env.development.local

.env.test.local

.env.production.local

# Database

db.sqlite3

FROM apache/airflow:3.0.5

COPY requirements.txt .

RUN pip install --upgrade pip

# switch to tsinghua pip/conda source

RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple/ \

    && pip config set global.extra-index-url "https://mirrors.bfsu.edu.cn/pypi/web/simple"

# install requirements

RUN pip install --no-cache-dir -r requirements.txt

all: down pull build init up

all: down pull init up

pull:

	git pull

mkdir:

	mkdir -p ./dags ./logs ./plugins ./config ./pg_data

	#echo -e "AIRFLOW_UID=$(id -u)" > .env

	mkdir -p ./dags ./volumes/logs ./volumes/plugins ./volumes/config ./volumes/pg_data

rmdir:

	rm -rf ./logs ./plugins ./config ./pg_data

build:

	docker build -t csst-airflow . --no-cache

	rm -rf ./volumes

init:

	docker compose up airflow-init

up:

	docker compose up -d

up-master:

	docker compose -f docker-compose.yaml --env-file .env --profile flower up --force-recreate

up-worker:

	docker compose -f docker-compose.yaml --env-file .env up worker --force-recreate

copy-env-csu:

	cp envs/.env.csu .env

copy-env-p368:

	cp envs/.env.p368 .env

copy-env-zjlab:

	cp envs/.env.zjlab .env

down:

	docker compose down

#

# Airflow 部署指南

## 

本指南描述如何使用官方 Apache Airflow 3.0.5 镜像部署 Airflow 服务器，使用 CeleryExecutor、Redis 和 PostgreSQL。

```shell

## 前提条件

- 已安装 Docker 和 Docker Compose

- Git

- Make

## 部署步骤

### 1. 克隆仓库

```bash

git clone <仓库地址>

cd csst-airflow/docker-celery-3.0.5

```

### 2. 配置环境

选择适合您设置的环境文件：

```bash

# 对于 CSU 环境

make copy-env-csu

# 对于 P368 环境

make copy-env-p368

# 对于 ZJLAB 环境

make copy-env-zjlab

```

### 3. 初始化目录结构

```bash

make mkdir

```

### 4. 初始化 Airflow

```bash

make init

```

### 5. 启动 Airflow 服务

#### 启动主服务（包括 Flower）

```bash

make up-master

```

#### 启动工作节点服务

```bash

make up-worker

```

#### 在后台启动所有服务

```bash

make up

```

## Makefile 命令

| 命令 | 描述 |

|------|------|

| `make all` | 完整部署：停止、拉取、初始化、启动 |

| `make up-master` | 启动 Airflow 主服务和 Flower |

| `make up-worker` | 启动 Airflow 工作节点 |

| `make copy-env-csu` | 复制 CSU 环境文件 |

| `make copy-env-p368` | 复制 P368 环境文件 |

| `make copy-env-zjlab` | 复制 ZJLAB 环境文件 |

| `make init` | 初始化 Airflow |

| `make up` | 在后台启动所有服务 |

| `make down` | 停止所有服务 |

| `make migrate` | 运行数据库迁移 |

| `make clean` | 清理服务 |

| `make ps` | 列出运行中的服务 |

| `make restart` | 重启服务 |

## 访问 Airflow

- Airflow Web UI: `http://<MASTER_IP>:38080`

- Flower (Celery 监控): `http://<MASTER_IP>:35555`

- PostgreSQL: `localhost:35432`

- Redis: `localhost:36379`

## 配置

部署使用官方 Apache Airflow 3.0.5 镜像，具有以下配置：

- CeleryExecutor 用于分布式任务执行

- PostgreSQL 作为元数据数据库

- Redis 作为消息代理

- 通过环境变量进行自定义 Airflow 配置

## 环境变量

.env 文件中的关键环境变量：

- `AIRFLOW_IMAGE_NAME`: 官方 Apache Airflow 3.0.5 镜像

- `AIRFLOW_UID`: Airflow 容器的用户 ID

- `MASTER_IP`: 主节点的 IP 地址

- `JWT_SECRET`: JWT 认证的密钥

- `_AIRFLOW_WWW_USER_USERNAME`: Airflow Web UI 用户名

- `_AIRFLOW_WWW_USER_PASSWORD`: Airflow Web UI 密码

## 常见操作

### 更新 Airflow

```bash

make down

# 如需更新环境文件

make up

```

### 运行数据库迁移

```bash

make migrate

```

### 检查服务状态

```bash

make ps

```

### 清理

```bash

make clean

```

## 故障排除

- 确保 Docker 正在运行

- 检查端口 38080、35555、35432 和 36379 是否可用

- 验证环境变量是否正确设置

- 检查容器日志以了解错误：

  ```bash

  docker compose logs

  ```

    AIRFLOW__CORE__FERNET_KEY: ''

    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'

    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'

    AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/'

    AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://${MASTER_IP}:38080/execution/'

    # yamllint disable rule:line-length

    # Use simple http server on scheduler for health checks

    # See https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/check-health.html#scheduler-health-check-server

    AIRFLOW__CORE__TEST_CONNECTION: 'Enabled'

    AIRFLOW__LOGGING__LOGGING_LEVEL: "INFO"

    # API

    AIRFLOW__API__HOST: "${MASTER_IP}"

    AIRFLOW__API__PORT: 38080

    AIRFLOW__API__HOST: "0.0.0.0"

    AIRFLOW__API__PORT: 8080

    AIRFLOW__API__WORKERS: 4

    # API AUTH

    AIRFLOW__API_AUTH__JWT_SECRET: "${JWT_SECRET}"  # 从 .env 注入

    AIRFLOW__SCHEDULER__IGNORE_FIRST_DEPENDS_ON_PAST_BY_DEFAULT: 'true'

    # CELERY FLOWER

    AIRFLOW__CELERY__FLOWER_BASIC_AUTH: "${_AIRFLOW_WWW_USER_USERNAME}:${_AIRFLOW_WWW_USER_PASSWORD}"

    AIRFLOW__CELERY__FLOWER_HOST: "${MASTER_IP}"

    AIRFLOW__CELERY__FLOWER_HOST: "0.0.0.0"

    AIRFLOW__CELERY__FLOWER_PORT: 35555

    AIRFLOW__CELERY__WORKER_CONCURRENCY: 64

    AIRFLOW__CELERY__WORKER_AUTOSCALE: "1,64"

    AIRFLOW__CELERY__WORKER_PREFETCH_MULTIPLIER: 1

  volumes:

    - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags

    - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs

    - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config

    - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins

    - ${AIRFLOW_PROJ_DIR:-.}/volumes/logs:/opt/airflow/logs

    - ${AIRFLOW_PROJ_DIR:-.}/volumes/config:/opt/airflow/config

    - ${AIRFLOW_PROJ_DIR:-.}/volumes/plugins:/opt/airflow/plugins

  user: "${AIRFLOW_UID:-50000}:0"

  depends_on:

    &airflow-common-depends-on

      POSTGRES_DB: airflow

    volumes:

#      - postgres-db-volume:/var/lib/postgresql/data

      - ${AIRFLOW_PROJ_DIR:-.}/pg_data:/var/lib/postgresql/data

      - ${AIRFLOW_PROJ_DIR:-.}/volumes/pg_data:/var/lib/postgresql/data

    healthcheck:

      test: ["CMD", "pg_isready", "-U", "airflow"]

      interval: 10s