kafka-cluster 구축 테스트 로그 #11

dnlpys · 2024-09-03T12:45:15Z

dnlpys
Sep 3, 2024
Maintainer

단일 인스턴스에서 컨테이너 3개 띄워서 클러스터 구축

dnlpys9999@datapipeline1:~/app/kafka1$ cat docker-compose.yaml

version: '3.8'
services:
  kafka-1:
    container_name: kafka-1
    image: confluentinc/cp-kafka:7.5.3
    platform: linux/amd64
    ports:
      - "${KAFKA_1_PORT}:9092"
    volumes:
      - /home/dnlpys9999/app/kafka_data/kafka-1:/var/lib/kafka/data
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:29092,EXTERNAL://10.178.0.5:${KAFKA_1_PORT}
      KAFKA_LISTENERS: INTERNAL://:29092,CONTROLLER://:29093,EXTERNAL://0.0.0.0:${KAFKA_1_PORT}
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      CLUSTER_ID: ${KAFKA_CLUSTER_ID:-MkU3OEVBNTcwNTJENDM2Qk}

  kafka-2:
    container_name: kafka-2
    image: confluentinc/cp-kafka:7.5.3
    platform: linux/amd64
    ports:
      - "${KAFKA_2_PORT}:9093"
    volumes:
      - /home/dnlpys9999/app/kafka_data/kafka-2:/var/lib/kafka/data
    environment:
      KAFKA_NODE_ID: 2
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:29092,EXTERNAL://10.178.0.5:${KAFKA_2_PORT}
      KAFKA_LISTENERS: INTERNAL://:29092,CONTROLLER://kafka-2:29093,EXTERNAL://0.0.0.0:${KAFKA_2_PORT}
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      CLUSTER_ID: ${KAFKA_CLUSTER_ID:-MkU3OEVBNTcwNTJENDM2Qk}

  kafka-3:
    container_name: kafka-3
    image: confluentinc/cp-kafka:7.5.3
    platform: linux/amd64
    ports:
      - "${KAFKA_3_PORT}:9094"
    volumes:
      - /home/dnlpys9999/app/kafka_data/kafka-3:/var/lib/kafka/data
    environment:
      KAFKA_NODE_ID: 3
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:29092,EXTERNAL://10.178.0.5:${KAFKA_3_PORT}
      KAFKA_LISTENERS: INTERNAL://:29092,CONTROLLER://kafka-3:29093,EXTERNAL://0.0.0.0:${KAFKA_3_PORT}
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:29093,2@kafka-2:29093,3@kafka-3:29093
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      CLUSTER_ID: ${KAFKA_CLUSTER_ID:-MkU3OEVBNTcwNTJENDM2Qk}

volumes: {}

인스턴스3개로 각 1개씩 컨테이너 띄워서 클러스터 구축

초기 docker-compose.yaml파일

# Kafka-1

version: '3.8'
services:
  kafka:
    image: confluentinc/cp-kafka:7.5.3
    hostname: kafka-1
    container_name: kafka-1
    ports:
      - "9092:9092"  # 브로커 포트
      - "9093:9093"  # 컨트롤러 포트
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://<인스턴스1 Private IP>:9092,CONTROLLER://<인스턴스1 Private IP>:9093
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@<인스턴스1 Private IP>:9093,2@<인스턴스2 Private IP>:9093,3@<인스턴스3 Private IP>:9093"
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LOG_DIRS: /var/lib/kafka/data
    volumes:
      - ~/kafka-data:/var/lib/kafka/data

# Kafka-2

version: '3.8'
services:
  kafka:
    image: confluentinc/cp-kafka:7.5.3
    hostname: kafka-2
    container_name: kafka-2
    ports:
      - "9092:9092"  # 브로커 포트
      - "9093:9093"  # 컨트롤러 포트
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://<인스턴스2 Private IP>:9092,CONTROLLER://<인스턴스2 Private IP>:9093
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@<인스턴스1 Private IP>:9093,2@<인스턴스2 Private IP>:9093,3@<인스턴스3 Private IP>:9093"
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LOG_DIRS: /var/lib/kafka/data
    volumes:
      - ~/kafka-data:/var/lib/kafka/data

#Kafka-3

version: '3.8'
services:
  kafka:
    image: confluentinc/cp-kafka:7.5.3
    hostname: kafka-3
    container_name: kafka-3
    ports:
      - "9092:9092"  # 브로커 포트
      - "9093:9093"  # 컨트롤러 포트
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://<인스턴스3 Private IP>:9092,CONTROLLER://<인스턴스3 Private IP>:9093
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@<인스턴스1 Private IP>:9093,2@<인스턴스2 Private IP>:9093,3@<인스턴스3 Private IP>:9093"
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LOG_DIRS: /var/lib/kafka/data
    volumes:
      - ~/kafka-data:/var/lib/kafka/data

docker compose up >> error !

dnlpys9999@datapipeline1:~/app/kafka$ docker logs kafka-1
===> User
uid=1000(appuser) gid=1000(appuser) groups=1000(appuser)
===> Configuring ...
Running in KRaft mode...
===> Running preflight checks ...
===> Check if /var/lib/kafka/data is writable ...
Command [/usr/local/bin/dub path /var/lib/kafka/data writable] FAILED !

# 문제해결
디렉토리 권한 확인
chmod 777 ~/app/kafka/kafka-data

docker compose up >> error !

dnlpys9999@datapipeline1:~/app/kafka$ docker compose up
WARN[0000] /home/dnlpys9999/app/kafka/docker-compose.yaml: the attribute version is obsolete, it will be ignored, please remove it to avoid potential confusion
[+] Running 2/1
 ✔ Network kafka_default  Created                                                                                                                                                             0.1s
 ✔ Container kafka-1      Created                                                                                                                                                             0.1s
Attaching to kafka-1
kafka-1  | ===> User
kafka-1  | uid=1000(appuser) gid=1000(appuser) groups=1000(appuser)
kafka-1  | ===> Configuring ...
kafka-1  | Running in KRaft mode...
kafka-1  | ===> Running preflight checks ...
kafka-1  | ===> Check if /var/lib/kafka/data is writable ...
kafka-1  | ===> Running in KRaft mode, skipping Zookeeper health check...
kafka-1  | ===> Using provided cluster id MkU3OEVBNTcwNTJENDM2Qk ...
kafka-1  | Exception in thread "main" org.apache.kafka.common.config.ConfigException: Failed to parse voter ID as an integer from 10.178.0.5 
at org.apache.kafka.raft.RaftConfig.parseVoterId(RaftConfig.java:205) at org.apache.kafka.raft.RaftConfig.parseVoterConnections(RaftConfig.java:218) 
at org.apache.kafka.raft.RaftConfig$ControllerQuorumVotersValidator.ensureValid(RaftConfig.java:270) 
at org.apache.kafka.common.config.ConfigDef.parseValue(ConfigDef.java:499) at org.apache.kafka.common.config.ConfigDef.parse(ConfigDef.java:483) 
at org.apache.kafka.common.config.AbstractConfig.<init>(AbstractConfig.java:112) at org.apache.kafka.common.config.AbstractConfig.<init>(AbstractConfig.java:145) 
at kafka.server.KafkaConfig.<init>(KafkaConfig.scala:1566) at kafka.server.KafkaConfig.<init>(KafkaConfig.scala:1568) 
at kafka.tools.StorageTool$.$anonfun$main$1(StorageTool.scala:50) at scala.Option.flatMap(Option.scala:283) 
at kafka.tools.StorageTool$.main(StorageTool.scala:50) at kafka.tools.StorageTool.main(StorageTool.scala)
kafka-1 exited with code 1

# 문제
Kafka의 Raft(즉, KRaft) 모드에서 컨트롤러 쿼럼 투표자 ID를 정수로 파싱하는 데 실패

# 문제해결
KAFKA_CONTROLLER_QUORUM_VOTERS 설정 확인
KAFKA_CONTROLLER_QUORUM_VOTERS: "1@10.178.0.5:9093,2@10.178.0.6:9093,3@10.178.0.7:9093"
이 환경설정이 ip가 누락되었음.

docker compose up >> error !

WARN[0000] /home/dnlpys9999/app/kafka/docker-compose.yaml: the attribute `version` is obsolete, it will be ignored, please remove it to avoid potential confusion
[+] Running 2/1
 ✔ Network kafka_default  Created                                                                                                                                                             0.1s
 ✔ Container kafka-1      Created                                                                                                                                                             0.1s
Attaching to kafka-1
kafka-1  | ===> User
kafka-1  | uid=1000(appuser) gid=1000(appuser) groups=1000(appuser)
kafka-1  | ===> Configuring ...
kafka-1  | Running in KRaft mode...
kafka-1  | ===> Running preflight checks ...
kafka-1  | ===> Check if /var/lib/kafka/data is writable ...
kafka-1  | ===> Running in KRaft mode, skipping Zookeeper health check...
kafka-1  | ===> Using provided cluster id MkU3OEVBNTcwNTJENDM2Qk ...
kafka-1  | Exception in thread "main" java.lang.IllegalArgumentException: Error creating broker listeners from 'PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093': No security protocol defined for listener CONTROLLER at kafka.utils.CoreUtils$.listenerListToEndPoints(CoreUtils.scala:254) at kafka.server.KafkaConfig.listeners(KafkaConfig.scala:2053) at kafka.server.KafkaConfig.<init>(KafkaConfig.scala:1772) at kafka.server.KafkaConfig.<init>(KafkaConfig.scala:1568) at kafka.tools.StorageTool$.$anonfun$main$1(StorageTool.scala:50) at scala.Option.flatMap(Option.scala:283) at kafka.tools.StorageTool$.main(StorageTool.scala:50) at kafka.tools.StorageTool.main(StorageTool.scala) Caused by: java.lang.IllegalArgumentException: No security protocol defined for listener CONTROLLER at kafka.cluster.EndPoint$.$anonfun$createEndPoint$2(EndPoint.scala:49) at scala.collection.immutable.Map$Map4.getOrElse(Map.scala:528) at kafka.cluster.EndPoint$.securityProtocol$1(EndPoint.scala:49) at kafka.cluster.EndPoint$.createEndPoint(EndPoint.scala:57) at kafka.utils.CoreUtils$.$anonfun$listenerListToEndPoints$6(CoreUtils.scala:251) at scala.collection.StrictOptimizedIterableOps.map(StrictOptimizedIterableOps.scala:100) at scala.collection.StrictOptimizedIterableOps.map$(StrictOptimizedIterableOps.scala:87) at scala.collection.mutable.ArraySeq.map(ArraySeq.scala:37) at kafka.utils.CoreUtils$.listenerListToEndPoints(CoreUtils.scala:251) ... 7 more
kafka-1 exited with code 1

# 문제
Kafka의 listeners 및 advertised.listeners 설정에서 CONTROLLER 리스너의 보안 프로토콜을 
명시해야 합니다. PLAINTEXT 프로토콜을 사용하여 보안 프로토콜을 설정할 수 있습니다.

# 문제해결
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP을 추가하여 CONTROLLER 리스너의 보안 프로토콜을 PLAINTEXT로 설정
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT

docker compose up >> error !

dnlpys9999@datapipeline1:~/app/kafka$ docker compose down
WARN[0000] /home/dnlpys9999/app/kafka/docker-compose.yaml: the attribute `version` is obsolete, it will be ignored, please remove it to avoid potential confusion
dnlpys9999@datapipeline1:~/app/kafka$ docker compose up
WARN[0000] /home/dnlpys9999/app/kafka/docker-compose.yaml: the attribute `version` is obsolete, it will be ignored, please remove it to avoid potential confusion
[+] Running 2/1
 ✔ Network kafka_default  Created                                                                                                                                                                                                                     0.1s
 ✔ Container kafka-1      Created                                                                                                                                                                                                                     0.1s
Attaching to kafka-1
kafka-1  | ===> User
kafka-1  | uid=1000(appuser) gid=1000(appuser) groups=1000(appuser)
kafka-1  | ===> Configuring ...
kafka-1  | Running in KRaft mode...
kafka-1  | ===> Running preflight checks ...
kafka-1  | ===> Check if /var/lib/kafka/data is writable ...
kafka-1  | ===> Running in KRaft mode, skipping Zookeeper health check...
kafka-1  | ===> Using provided cluster id MkU3OEVBNTcwNTJENDM2Qk ...
kafka-1  | Exception in thread "main" java.lang.IllegalArgumentException: requirement failed: controller.listener.names must contain at least one value appearing in the 'listeners' configuration when running the KRaft controller role at scala.Predef$.require(Predef.scala:337) at kafka.server.KafkaConfig.validateControllerListenerExistsForKRaftController$1(KafkaConfig.scala:2227) at kafka.server.KafkaConfig.validateValues(KafkaConfig.scala:2289) at kafka.server.KafkaConfig.<init>(KafkaConfig.scala:2160) at kafka.server.KafkaConfig.<init>(KafkaConfig.scala:1568) at kafka.tools.StorageTool$.$anonfun$main$1(StorageTool.scala:50) at scala.Option.flatMap(Option.scala:283) at kafka.tools.StorageTool$.main(StorageTool.scala:50) at kafka.tools.StorageTool.main(StorageTool.scala)
kafka-1 exited with code 1

# 문제
현재 로그에서 발생하는 오류는 controller.listener.names가 listeners 설정에 포함된 값으로 설정되어야 한다는 것입니다. Kafka는 KRaft 모드에서 CONTROLLER 리스너를 사용하려면 controller.listener.names에 CONTROLLER 리스너가 포함되어야 합니다.

이 문제를 해결하기 위해 KAFKA_LISTENER_NAMES 환경 변수를 설정하여 CONTROLLER 리스너가 controller.listener.names에 포함되도록 해야 합니다.

# 문제해결
KAFKA_LISTENER_NAMES을 추가
KAFKA_LISTENER_NAMES: PLAINTEXT,CONTROLLER

docker compose up >> error !

dnlpys9999@datapipeline1:~/app/kafka$ dnlpys9999@datapipeline1:~/app/kafka$ docker compose up
WARN[0000] /home/dnlpys9999/app/kafka/docker-compose.yaml: the attribute `version` is obsolete, it will be ignored, please remove it to avoid potential confusion
[+] Running 2/1
 ✔ Network kafka_default  Created                                                                                                                                                                                                                     0.1s
 ✔ Container kafka-1      Created                                                                                                                                                                                                                     0.1s
Attaching to kafka-1
kafka-1  | ===> User
kafka-1  | uid=1000(appuser) gid=1000(appuser) groups=1000(appuser)
kafka-1  | ===> Configuring ...
kafka-1  | Running in KRaft mode...
kafka-1  | ===> Running preflight checks ...
kafka-1  | ===> Check if /var/lib/kafka/data is writable ...
kafka-1  | ===> Running in KRaft mode, skipping Zookeeper health check...
kafka-1  | ===> Using provided cluster id MkU3OEVBNTcwNTJENDM2Qk ...
kafka-1  | Exception in thread "main" java.lang.IllegalArgumentException: requirement failed: controller.listener.names must contain at least one value appearing in the 'listeners' configuration when running the KRaft controller role at scala.Predef$.require(Predef.scala:337) at kafka.server.KafkaConfig.validateControllerListenerExistsForKRaftController$1(KafkaConfig.scala:2227) at kafka.server.KafkaConfig.validateValues(KafkaConfig.scala:2289) at kafka.server.KafkaConfig.<init>(KafkaConfig.scala:2160) at kafka.server.KafkaConfig.<init>(KafkaConfig.scala:1568) at kafka.tools.StorageTool$.$anonfun$main$1(StorageTool.scala:50) at scala.Option.flatMap(Option.scala:283) at kafka.tools.StorageTool$.main(StorageTool.scala:50) at kafka.tools.StorageTkafka-1 exited with code 1)

# 문제
현재 로그에서 발생하는 오류는 controller.listener.names가 listeners 설정에 포함된 값으로 설정되어야 한다는 것입니다. 이 오류는 CONTROLLER 리스너가 listeners 설정에서 명시적으로 지정되지 않아서 발생합니다.

이 문제를 해결하기 위해 KAFKA_LISTENER_NAMES와 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP을 올바르게 설정하여 CONTROLLER 리스너가 포함되도록 해야 합니다. 또한, Kafka 7.5.3에서는 KAFKA_LISTENER_NAMES와 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP을 올바르게 설정해도 CONTROLLER 리스너가 요구 사항을 충족하지 못할 수 있습니다. 이 경우, controller.listener.names를 명시적으로 설정하는 것이 필요합니다.

# 문제해결
# controller.listener.names를 명시적으로 설정하는 것이 필요
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER  # 추가

docker compose up >> error !

dnlpys9999@datapipeline1:~/app/kafka$ docker compose up
WARN[0000] /home/dnlpys9999/app/kafka/docker-compose.yaml: the attribute version is obsolete, it will be ignored, please remove it to avoid potential confusion
[+] Running 2/1
 ✔ Network kafka_default  Created                                                                                                                                                                                                                     0.1s
 ✔ Container kafka-1      Created                                                                                                                                                                                                                     0.0s
Attaching to kafka-1
kafka-1  | ===> User
kafka-1  | uid=1000(appuser) gid=1000(appuser) groups=1000(appuser)
kafka-1  | ===> Configuring ...
kafka-1  | Running in KRaft mode...
kafka-1  | ===> Running preflight checks ...
kafka-1  | ===> Check if /var/lib/kafka/data is writable ...
kafka-1  | ===> Running in KRaft mode, skipping Zookeeper health check...
kafka-1  | ===> Using provided cluster id MkU3OEVBNTcwNTJENDM2Qk ...
kafka-1  | Exception in thread "main" java.lang.IllegalArgumentException: requirement failed: The advertised.listeners config must not contain KRaft controller listeners from controller.listener.names when process.roles contains the broker role because Kafka clients that send requests via advertised listeners do not send requests to KRaft controllers -- they only send requests to KRaft brokers. at scala.Predef$.require(Predef.scala:337) at kafka.server.KafkaConfig.validateAdvertisedListenersDoesNotContainControllerListenersForKRaftBroker$1(KafkaConfig.scala:2219) at kafka.server.KafkaConfig.validateValues(KafkaConfig.scala:2287) at kafka.server.KafkaConfig.<init>(KafkaConfig.scala:2160) at kafka.server.KafkaConfig.<init>(KafkaConfig.scala:1568) at kafka.tools.StorageTool$.$anonfun$main$1(StorageTool.scala:50) at scala.Option.flatMap(Option.scala:283) at kafka.tools.StorageTool$.main(StorageTool.scala:50) at kafka.tools.StorageTool.main(StorageTool.scala)
kafka-1 exited with code 1

# 문제
현재 로그에서의 오류는 advertised.listeners 구성에서 KRaft 컨트롤러 리스너가 포함되어서는 
안 된다는 것입니다. advertised.listeners는 Kafka 클라이언트가 브로커에 연결할 때 사용하는 
주소를 제공하며, KRaft 컨트롤러 리스너는 클라이언트의 요청을 받지 않기 때문에 
advertised.listeners에 포함될 수 없습니다.

# 문제 해결
advertised.listeners에서 CONTROLLER 리스너를 제거합니다. advertised.listeners는 브로커가 클라이언트와 통신하는 데 사용되므로, 컨트롤러 리스너는 제거해야 합니다.
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.178.0.5:9092

컨테이너 안에서

# 접속
docker exec -it kafka-* /bin/bash

# 토픽 생성
kafka-topics --create --topic my-topic --bootstrap-server localhost:9092 --partitions 1 --replication-factor 1

# 토픽 확인
kafka-topics --list --bootstrap-server localhost:9092

# kafka-1 producing
kafka-console-producer --topic my-topic --bootstrap-server localhost:9092

# kafka-2 consuming
kafka-console-consumer --topic my-topic --bootstrap-server localhost:9092 --from-beginning

네트워크 체크

# kafka-1 > kafka-2
telnet 10.178.0.6 9092
or
nc -zv 10.178.0.6 9092

# kafka-2 > kafka-1
telnet 10.178.0.5 9092
or
nc -zv 10.178.0.5 9092

docker-compose.yaml

# kafka-1
dnlpys9999@datapipeline1:~/app/kafka$ cat docker-compose.yaml
version: '3.8'
services:
  kafka:
    image: confluentinc/cp-kafka:7.5.3
    hostname: kafka-1
    container_name: kafka-1
    ports:
      - "9092:9092"  # 브로커 포트
      - "9093:9093"  # 컨트롤러 포트
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.178.0.5:9092
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@10.178.0.5:9093,2@10.178.0.6:9093,3@10.178.0.7:9093"
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: ${KAFKA_CLUSTER_ID:-MkU3OEVBNTcwNTJENDM2Qk}
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_LISTENER_NAMES: PLAINTEXT,CONTROLLER
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER  # 추가
    volumes:
      - /home/dnlpys9999/app/kafka-data:/var/lib/kafka/data
      
     
# kafka-2
dnlpys9999@datapipeline2:~/app/kafka$ cat docker-compose.yaml
version: '3.8'
services:
  kafka:
    image: confluentinc/cp-kafka:7.5.3
    hostname: kafka-2
    container_name: kafka-2
    ports:
      - "9092:9092"  # 브로커 포트
      - "9093:9093"  # 컨트롤러 포트
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.178.0.6:9092
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@10.178.0.5:9093,2@10.178.0.6:9093,3@10.178.0.7:9093"
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: ${KAFKA_CLUSTER_ID:-MkU3OEVBNTcwNTJENDM2Qk}
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_LISTENER_NAMES: PLAINTEXT,CONTROLLER
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER  # 추가
    volumes:
      - /home/dnlpys9999/app/kafka-data:/var/lib/kafka/data
      
# kafka-3
dnlpys9999@datapipeline3:~/app/kafka$ cat docker-compose.yaml
version: '3.8'
services:
  kafka:
    image: confluentinc/cp-kafka:7.5.3
    hostname: kafka-3
    container_name: kafka-3
    ports:
      - "9092:9092"  # 브로커 포트
      - "9093:9093"  # 컨트롤러 포트
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://10.178.0.7:9092
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@10.178.0.5:9093,2@10.178.0.6:9093,3@10.178.0.7:9093"
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LOG_DIRS: /var/lib/kafka/data
      CLUSTER_ID: ${KAFKA_CLUSTER_ID:-MkU3OEVBNTcwNTJENDM2Qk}
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_LISTENER_NAMES: PLAINTEXT,CONTROLLER
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER  # 추가
    volumes:
      - /home/dnlpys9999/app/kafka-data:/var/lib/kafka/data

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

kafka-cluster 구축 테스트 로그 #11

{{title}}

Replies: 0 comments

Select a reply

kafka-cluster 구축 테스트 로그 #11

dnlpys Sep 3, 2024 Maintainer

단일 인스턴스에서 컨테이너 3개 띄워서 클러스터 구축

인스턴스3개로 각 1개씩 컨테이너 띄워서 클러스터 구축

초기 docker-compose.yaml파일

docker compose up >> error !

docker compose up >> error !

docker compose up >> error !

docker compose up >> error !

docker compose up >> error !

docker compose up >> error !

컨테이너 안에서

네트워크 체크

docker-compose.yaml

Replies: 0 comments

dnlpys
Sep 3, 2024
Maintainer