Commit cc83a85b authored by BO ZHANG's avatar BO ZHANG 🏀
Browse files

feat(logging): add elasticsearch logging support

parent 76ec8e70

.gitignore

+8 −0
Original line number Diff line number Diff line
@@ -27,6 +27,14 @@ logs/ 
.DS_Store

Thumbs.db



# Trae IDE

.trae



# Directories

conda/

docker/

python/



# Build artifacts

build/

dist/

docker-celery-3.0.5/config/filebeat/filebeat.yml

0 → 100644
+22 −0
Original line number Diff line number Diff line 
filebeat.inputs:

- type: filestream

  id: airflow-logs

  enabled: true

  paths:

    - /usr/share/filebeat/logs/*/*/*/*.log

  # Airflow logs often have multi-line stack traces

  parsers:

    - multiline:

        type: pattern

        pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'

        negate: true

        match: after



output.elasticsearch:

  hosts: ["elasticsearch:9200"]

  # Airflow expects indices to follow this pattern if we use default configuration

  index: "airflow-%{+yyyy.MM.dd}"



setup.template.name: "airflow"

setup.template.pattern: "airflow-*"

setup.ilm.enabled: false

docker-celery-3.0.5/docker-compose.yaml

+48 −0
Original line number Diff line number Diff line
@@ -101,6 +101,11 @@ x-airflow-common: 
    AIRFLOW__API_AUTH__JWT_SECRET: "${JWT_SECRET}"  # 从 .env 注入

    AIRFLOW__API_AUTH__JWT_ALGORITHM: "HS512"

    AIRFLOW__API_AUTH__JWT_EXPIRATION_TIME: 86400000 # 1000 days

    # Elasticsearch Remote Logging

    AIRFLOW__LOGGING__REMOTE_LOGGING: 'true'

    AIRFLOW__ELASTICSEARCH__HOST: 'http://elasticsearch:9200'

    AIRFLOW__ELASTICSEARCH__WRITE_STDOUT: 'true'

    AIRFLOW__ELASTICSEARCH__JSON_FORMAT: 'false'

    # Operators

    AIRFLOW__OPERATORS__DEFAULT_CPUS: 1

    AIRFLOW__OPERATORS__DEFAULT_RAM: 1024
@@ -167,6 +172,49 @@ services: 
      start_period: 30s

    restart: always



  elasticsearch:

    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.1

    environment:

      - discovery.type=single-node

      - xpack.security.enabled=false

      - ES_JAVA_OPTS=-Xms1g -Xmx1g

    ports:

      - "39200:9200"  # 外部端口大于 30000

    restart: always

    volumes:

      - ${AIRFLOW_PROJ_DIR:-.}/volumes/es_data:/usr/share/elasticsearch/data

    healthcheck:

      test: ["CMD-SHELL", "curl -s http://localhost:9200 >/dev/null || exit 1"]

      interval: 30s

      timeout: 10s

      retries: 50

      start_period: 30s



  kibana:

    image: docker.elastic.co/kibana/kibana:8.11.1

    environment:

      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200

    ports:

      - "35601:5601"  # 外部端口大于 30000

    restart: always

    depends_on:

      elasticsearch:

        condition: service_healthy



  filebeat:

    image: docker.elastic.co/beats/filebeat:8.11.1

    user: root

    environment:

      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200

    volumes:

      - ${AIRFLOW_PROJ_DIR:-.}/volumes/logs:/usr/share/filebeat/logs:ro

      - ${AIRFLOW_PROJ_DIR:-.}/config/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro

    command: filebeat -e -strict.perms=false

    restart: always

    depends_on:

      elasticsearch:

        condition: service_healthy



  airflow-apiserver:

    <<: *airflow-common

    command: api-server