From 1b3fea740a8c44cf9574ec649b3509a245ea9604 Mon Sep 17 00:00:00 2001 From: rmoff Date: Wed, 8 May 2024 10:02:27 +0100 Subject: [PATCH 1/2] Add kafka ngrok example --- README.md | 1 + kafka-ngrok/README.adoc | 30 +++++++++++++++ kafka-ngrok/docker-compose.yml | 69 ++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 kafka-ngrok/README.adoc create mode 100644 kafka-ngrok/docker-compose.yml diff --git a/README.md b/README.md index a28286a..2e0cd2a 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ This repository contains examples of use cases that utilize Decodable streaming |[Kafka / Flink / Iceberg](kafka-iceberg)| Integrating Apache Kafka with Apache Iceberg through Apache Flink. As presented at Kafka Summit London 2024| |[Flink SQL Troubleshooting](troubleshooting-flinksql)| A set of Docker Compose environments for demonstrating various Flink SQL troubleshooting scenarios (see [related blog](https://www.decodable.co/blog/flink-sql-misconfiguration-misunderstanding-and-mishaps?utm_medium=github&utm_source=examples_repo&utm_campaign=blog&utm_content=troubleshooting-flinksql))| |[Array Aggregation](array-agg)| Using the `array_agg()` UDF for denormalizing data in a pipeline from MySQL to OpenSearch | +|[Kafka with ngrok](kafka-ngrok)| Docker Compose for running Apache Kafka locally, accessible from the internet using ngrok| ## License diff --git a/kafka-ngrok/README.adoc b/kafka-ngrok/README.adoc new file mode 100644 index 0000000..12fe75c --- /dev/null +++ b/kafka-ngrok/README.adoc @@ -0,0 +1,30 @@ += Docker Compose for running Apache Kafka locally, accessible from the internet using ngrok + +image::https://rmoff.net/images/2023/11/ngrok02.webp[Overview diagram] + +_For details of how this works, see https://rmoff.net/2023/11/01/using-apache-kafka-with-ngrok/[this blog] (and take note of https://rmoff.net/2024/05/03/ngrok-dns-headaches/[possible problems] you might encounter)._ + +To use this you need to create an ngrok account and add a file called `.env` in this folder with the following entry: + +[source,bash] +---- +NGROK_AUTH_TOKEN= +---- + +Bring up the Kafka and ngrok stack with + +[source,bash] +---- +docker compose up +---- + +Once up, find out your Kafka broker host/post that is available on the internet: + +[source,bash] +---- +curl -s localhost:4040/api/tunnels | jq -r '.tunnels[0].public_url' | sed 's/tcp:\/\///g' +---- + +WARNING: There is **NO SECURITY** defined on this Kafka cluster. Anyone scanning the ngrok address/port may find this and be able to access your data. + +https://github.com/edenhill/kcat[kcat] is included. diff --git a/kafka-ngrok/docker-compose.yml b/kafka-ngrok/docker-compose.yml new file mode 100644 index 0000000..7ac4923 --- /dev/null +++ b/kafka-ngrok/docker-compose.yml @@ -0,0 +1,69 @@ +--- +version: '3.8' +services: + ngrok: + image: ngrok/ngrok:latest + container_name: ngrok + # Sign up for an ngrok account at https://dashboard.ngrok.com/signup + # Get your auth-token from https://dashboard.ngrok.com/get-started/your-authtoken + # and either put it directly in the file here, or write it to a .env file in + # the same folder as this Docker Compose file in the form + # NGROK_AUTH_TOKEN=your_token_value + command: tcp broker:9092 --log stdout --authtoken $NGROK_AUTH_TOKEN + ports: + - 4040:4040 # Web dashboard for ngrok + + zookeeper: + image: confluentinc/cp-zookeeper:7.5.1 + container_name: zookeeper + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + + broker: + image: confluentinc/cp-kafka:7.5.1 + container_name: broker + depends_on: + - zookeeper + - ngrok + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: DOCKER://broker:29092, NGROK://broker:9092 + KAFKA_ADVERTISED_LISTENERS: DOCKER://broker:29092 + KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER:PLAINTEXT,NGROK:PLAINTEXT + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + entrypoint: + - /bin/sh + - -c + - | + echo "Waiting for ngrok tunnel to be created" + while : ; do + curl_status=$$(curl -s -o /dev/null -w %{http_code} http://ngrok:4040/api/tunnels/command_line) + echo -e $$(date) "\tTunnels API HTTP state: " $$curl_status " (waiting for 200)" + if [ $$curl_status -eq 200 ] ; then + break + fi + sleep 5 + done + echo "ngrok tunnel is up" + # My kingdom for a jq 😿 + NGROK_LISTENER=$(curl -s http://ngrok:4040/api/tunnels/command_line | grep -Po '"public_url":.*?[^\\]",' | cut -d':' -f2- | tr -d ',"' | sed 's/tcp:\/\//NGROK:\/\//g') + echo $$NGROK_LISTENER + export KAFKA_ADVERTISED_LISTENERS="$$KAFKA_ADVERTISED_LISTENERS, $$NGROK_LISTENER" + echo "KAFKA_ADVERTISED_LISTENERS is set to " $$KAFKA_ADVERTISED_LISTENERS + /etc/confluent/docker/run + + kcat: + image: edenhill/kcat:1.7.1 + container_name: kcat + restart: no + entrypoint: tail -f /dev/null + +networks: + default: + name: zaphod From eb3cc91664df5deeb1f897deeb77dfe0adfcfc75 Mon Sep 17 00:00:00 2001 From: rmoff Date: Wed, 8 May 2024 10:03:30 +0100 Subject: [PATCH 2/2] Add link to ngrok signup --- kafka-ngrok/README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka-ngrok/README.adoc b/kafka-ngrok/README.adoc index 12fe75c..513d3f6 100644 --- a/kafka-ngrok/README.adoc +++ b/kafka-ngrok/README.adoc @@ -4,7 +4,7 @@ image::https://rmoff.net/images/2023/11/ngrok02.webp[Overview diagram] _For details of how this works, see https://rmoff.net/2023/11/01/using-apache-kafka-with-ngrok/[this blog] (and take note of https://rmoff.net/2024/05/03/ngrok-dns-headaches/[possible problems] you might encounter)._ -To use this you need to create an ngrok account and add a file called `.env` in this folder with the following entry: +To use this you need to https://dashboard.ngrok.com/signup[create an ngrok account] and add a file called `.env` in this folder with the following entry: [source,bash] ----