docker-compose.yaml

services:
  kafka-base:
    build:
      context: kafka
      dockerfile: Dockerfile
    image: local/kafka-base:v3.5.0

  kafka1:
    container_name: kafka1
    image: local/kafka-base:v3.5.0
    command: [ "sh", "-c", "/opt/kafka/bin/custom-start.sh" ]
    environment:
      # Below UUID is generated from https://uuid.rocks/short
      CLUSTER_ID: ${CLUSTER_ID:-bPpdXy8F6e7P49b2LCPH69}
      CUSTOM_NODE_ID: 1
      CUSTOM_LOG_DIRS: "/opt/kafka-kraft-data"
      CUSTOM_LISTENERS: "BROKER://:19092,CONTROLLER://:19093"
      CUSTOM_ROLES: "broker,controller"
      CUSTOM_CONTROLLER_QUORUM_VOTERS: "1@kafka1:19093,2@kafka2:29093,3@kafka3:39093"
      CUSTOM_INTER_BROKER_LISTENER_NAME: "BROKER"
      JMX_PORT: 19999
      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
      KAFKA_OPTS: "-javaagent:/opt/kafka/jmx_prometheus_javaagent.jar=17071:/opt/kafka/jmx-prom-java-agent-config.yaml"
    ports:
      - 18092:18092
      - 19092:19092
      - 17071:17071
    expose:
      - "18092"
      - "19092"
      - "17071"

  kafka2:
    container_name: kafka2
    image: local/kafka-base:v3.5.0
    command: [ "sh", "-c", "/opt/kafka/bin/custom-start.sh" ]
    environment:
      # Below UUID is generated from https://uuid.rocks/short
      CLUSTER_ID: ${CLUSTER_ID:-bPpdXy8F6e7P49b2LCPH69}
      CUSTOM_NODE_ID: 2
      CUSTOM_LOG_DIRS: "/opt/kafka-kraft-data"
      CUSTOM_LISTENERS: "BROKER://:29092,CONTROLLER://:29093"
      CUSTOM_ROLES: "broker,controller"
      CUSTOM_CONTROLLER_QUORUM_VOTERS: "1@kafka1:19093,2@kafka2:29093,3@kafka3:39093"
      CUSTOM_INTER_BROKER_LISTENER_NAME: "BROKER"
      JMX_PORT: 29999
      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
      KAFKA_OPTS: "-javaagent:/opt/kafka/jmx_prometheus_javaagent.jar=27071:/opt/kafka/jmx-prom-java-agent-config.yaml"
    ports:
      - 28092:28092
      - 29092:29092
      - 27071:27071
    expose:
      - "28092"
      - "29092"
      - "27071"

  kafka3:
    container_name: kafka3
    image: local/kafka-base:v3.5.0
    command: [ "sh", "-c", "/opt/kafka/bin/custom-start.sh" ]
    environment:
      # Below UUID is generated from https://uuid.rocks/short
      CLUSTER_ID: ${CLUSTER_ID:-bPpdXy8F6e7P49b2LCPH69}
      CUSTOM_NODE_ID: 3
      CUSTOM_LOG_DIRS: "/opt/kafka-kraft-data"
      CUSTOM_LISTENERS: "BROKER://:39092,CONTROLLER://:39093"
      CUSTOM_ROLES: "broker,controller"
      CUSTOM_CONTROLLER_QUORUM_VOTERS: "1@kafka1:19093,2@kafka2:29093,3@kafka3:39093"
      CUSTOM_INTER_BROKER_LISTENER_NAME: "BROKER"
      JMX_PORT: 39999
      KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
      KAFKA_OPTS: "-javaagent:/opt/kafka/jmx_prometheus_javaagent.jar=37071:/opt/kafka/jmx-prom-java-agent-config.yaml"
    ports:
      - 38092:38092
      - 39092:39092
      - 37071:37071
    expose:
      - "38092"
      - "39092"
      - "37071"

  kafka-ui:
    container_name: kafka-ui
    image: provectuslabs/kafka-ui:v0.7.1
    ports:
      - 8080:8080
    depends_on:
      - kafka1
      - schema-registry
    environment:
      KAFKA_CLUSTERS_0_NAME: kraft
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: "kafka1:19092,kafka2:29092,kafka3:39092"
      KAFKA_CLUSTERS_0_METRICS_PORT: 19999
      KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry:8081
      DYNAMIC_CONFIG_ENABLED: 'true' # not necessary, added for tests
      KAFKA_CLUSTERS_0_AUDIT_TOPICAUDITENABLED: 'true'
      KAFKA_CLUSTERS_0_AUDIT_CONSOLEAUDITENABLED: 'true'

  schema-registry:
    image: confluentinc/cp-schema-registry:7.4.0
    hostname: schema-registry
    container_name: schema-registry
    depends_on:
      - kafka1
      - kafka2
      - kafka3
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka1:19092,kafka2:29092,kafka3:39092'
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081

  kafka-client:
    build:
      context: clients
      dockerfile: Dockerfile
    image: local/kafka-client:v2.1.1

  producer:
    image: local/kafka-client:v2.1.1
    container_name: kafka-producer
    volumes:
      - ./clients/producer.py:/app/producer.py:ro
      - ./clients/protobuf:/app/protobuf:ro
    environment:
      BROKERS: 'kafka1:19092,kafka2:29092,kafka3:39092'
      TOPIC: test
      SCHEMA_REGISTRY_HOST: 'http://schema-registry:8081'
      MESSAGES_COUNT: '500'
    depends_on:
      - schema-registry
    command: [ "sh", "-c", "python3 /app/producer.py" ]

  consumer:
    image: local/kafka-client:v2.1.1
    container_name: kafka-consumer
    volumes:
      - ./clients/consumer.py:/app/consumer.py
      - ./clients/protobuf:/app/protobuf
    environment:
      BROKERS: 'kafka1:19092,kafka2:29092,kafka3:39092'
      TOPIC: test
      CONSUMER_GROUP: test
      CONSUMER_INSTANCE_ID: 1
    command: [ "sh", "-c", "python3 /app/consumer.py" ]