docker-compose.yml

version: '3'

services:

  jobmanager:
    container_name: jobmanager
    build:
      context: ./containers/flink/
    ports:
      - "8081:8081"
      - "9249:9249"
    command: jobmanager
    volumes:
      - ./code:/opt/flink/code
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager        

  taskmanager:
    container_name: taskmanager
    build:
      context: ./containers/flink/
    depends_on:
      - jobmanager
    command: taskmanager
    ports:
      - "9250:9249"
    volumes:
      - ./code:/opt/flink/code
    scale: 1
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 2


  zookeeper:
    image: bitnami/zookeeper:3.9.1
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
    volumes:
      - zookeeper_data:/var/lib/zookeeper/data
      - zookeeper_datalog:/var/lib/zookeeper/datalog

  kafka:
   image: docker.io/bitnami/kafka:3.4.1
   container_name: kafka
   ports:
     - "9094:9094"
   environment:
     - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
     - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
     - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
     - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
     - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094
     - ALLOW_PLAINTEXT_LISTENER='yes'
   volumes:
     - kafka:/var/lib/kafka/data
   depends_on:
     - zookeeper 

  postgres:
    image: postgres:latest
    container_name: postgres
    environment:
      - POSTGRES_PASSWORD=postgres
    ports:
      - "5432:5432"
    volumes:
      - ./postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
      - postgres:/var/lib/postgres/data

  fake_data:
    container_name: fake_data
    build:
      context: ./containers/fake_data/
    command: python /opt/fake_data/gen_fake_data.py
    volumes:
      - ./containers/fake_data:/opt/fake_data
      - ./code:/opt/fake_data/code
    restart: on-failure
    depends_on:
      - postgres
      - kafka

  redis:
    container_name: redis
    image: 'bitnami/redis:latest'
    environment:
      - ALLOW_EMPTY_PASSWORD=yes
    volumes:
      - redis:/var/lib/redis/data

  airflow:
    container_name: airflow
    image: bitnami/airflow:latest
    environment:
      - AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
      - AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08=
      - AIRFLOW_EXECUTOR=CeleryExecutor
      - AIRFLOW_DATABASE_NAME=postgres
      - AIRFLOW_DATABASE_USER=postgres
      - AIRFLOW_DATABASE_USERNAME=postgres
      - AIRFLOW_DATABASE_PASSWORD=postgres
      - AIRFLOW_USERNAME=user
      - AIRFLOW_EMAIL=laszlo@hey.com
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/postgres
    ports:
      - '8080:8080'
    volumes:
      - ./containers/airflow/dags:/opt/bitnami/airflow/dags
      - ./containers/airflow/includes:/opt/bitnami/airflow/includes
      - ./containers/airflow/logs:/opt/bitnami/airflow/logs
      - ./containers/airflow/plugins:/opt/bitnami/airflow/plugins
    depends_on:
      - postgres

  airflow-worker:
    container_name: airflow-worker
    image: bitnami/airflow-worker:latest
    environment:
      - AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
      - AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08=
      - AIRFLOW_EXECUTOR=CeleryExecutor
      - AIRFLOW_DATABASE_NAME=postgres
      - AIRFLOW_DATABASE_USER=postgres
      - AIRFLOW_DATABASE_USERNAME=postgres
      - AIRFLOW_DATABASE_PASSWORD=postgres
      - AIRFLOW_DATABASE_HOST=postgres
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/postgres
      - AIRFLOW_WEBSERVER_HOST=airflow
      - AIRFLOW_LOAD_EXAMPLES=no
    volumes:
      - ./containers/airflow/dags:/opt/bitnami/airflow/dags
      - ./containers/airflow/logs:/opt/bitnami/airflow/logs
      - ./containers/airflow/plugins:/opt/bitnami/airflow/plugins
      - ./containers/airflow/includes:/opt/bitnami/airflow/includes
    depends_on:
      - postgres
      - redis
  airflow-scheduler:
    container_name: airflow-scheduler
    image: bitnami/airflow-scheduler:latest
    environment:
      - AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
      - AIRFLOW_SECRET_KEY=a25mQ1FHTUh3MnFRSk5KMEIyVVU2YmN0VGRyYTVXY08=
      - AIRFLOW_EXECUTOR=CeleryExecutor
      - AIRFLOW_DATABASE_NAME=postgres
      - AIRFLOW_DATABASE_USER=postgres
      - AIRFLOW_DATABASE_USERNAME=postgres
      - AIRFLOW_DATABASE_PASSWORD=postgres
      - AIRFLOW_DATABASE_HOST=postgres
      - AIRFLOW_LOAD_EXAMPLES=no
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://postgres:postgres@postgres:5432/postgres
      - AIRFLOW_WEBSERVER_HOST=airflow
    volumes:
      - ./containers/airflow/dags:/opt/bitnami/airflow/dags
      - ./containers/airflow/logs:/opt/bitnami/airflow/logs
      - ./containers/airflow/plugins:/opt/bitnami/airflow/plugins
      - ./containers/airflow/includes:/opt/bitnami/airflow/includes
    depends_on:
      - postgres
      - redis

volumes:
  zookeeper_data:
  zookeeper_datalog:
  kafka:
  postgres:
  redis:
  airflow: