dataflow-compose.yml

version: '3'
# Configuration environment variables:
# - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use.
# - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register.
# - DATAFLOW_HOST_MOUNT_PATH and DATAFLOW_DOCKER_MOUNT_PATH are used to set the host and docker mount folders.
#   If not set DATAFLOW_HOST_MOUNT_PATH defaults to the local host folder where the docker compose is being started.
#   If not set DATAFLOW_DOCKER_MOUNT_PATH defaults to /home/cnb/scdf on dataflow-server and skipper containers.
#   Example to mount the local Maven repository: DATAFLOW_HOST_MOUNT_PATH=~/.m2 DATAFLOW_DOCKER_MOUNT_PATH=/home/cnb/.m2  docker-compose up
# - APPS_PORT_RANGE allows you to override the port range exported by the Skipper server. By default the APPS_PORT_RANGE
#   must supersede the local deployer's PORTRANGE_LOW and PORTRANGE_HIGH - the ports assigned to the stream apps run inside the Skipper container.
#   The stream apps tha run in their own docker containers must set [DOCKER_PORTRANGE_LOW, DOCKER_PORTRANGE_HIGH] range that
#   doesn't overlap with the APPS_PORT_RANGE.
#
# Exposed container ports:
# - 9393:9393               - Data Flow server port (http://localhost:9393/dashboard)
# - 7577:7577               - Skipper server port (http://localhost:7577/api)
# - 20000-20105:20000-20105 - Port range for all deployed stream applications that run inside the Skipper container (e.g. maven:// registered apps).
#                             That means you can reach the application's actuator endpoints from your host machine.
#                             The deployed stream applications that run in their own docker containers (e.g. docker:// registered apps),
#                             can be reached on the ports they expose.
services:

  dataflow-server:
    image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-2.10.2-SNAPSHOT}${BP_JVM_VERSION:-}
    container_name: dataflow-server
    user: root
    ports:
      - "9393:9393"
    environment:
      - LANG=en_US.utf8
      - LC_ALL=en_US.utf8
      - JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
      - SPRINGDOC_APIDOCS_ENABLED=true
      - SPRINGDOC_SWAGGERUI_ENABLED=true
      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_CLOUD_STREAM_KAFKA_STREAMS_BINDER_BROKERS=PLAINTEXT://kafka-broker:9092
      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_STREAM_SPRING_KAFKA_STREAMS_PROPERTIES_METRICS_RECORDING_LEVEL=DEBUG
      # Set CLOSECONTEXTENABLED=true to ensure that the CRT launcher is closed.
      - SPRING_CLOUD_DATAFLOW_APPLICATIONPROPERTIES_TASK_SPRING_CLOUD_TASK_CLOSECONTEXTENABLED=true
      - SPRING_CLOUD_SKIPPER_CLIENT_SERVER_URI=${SKIPPER_URI:-http://skipper-server:7577}/api
      # (Optionally) authenticate the default Docker Hub access for the App Metadata access.
      - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
      - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRY_CONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
      - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_USER=${METADATA_DEFAULT_DOCKERHUB_USER}
      - SPRING_CLOUD_DATAFLOW_CONTAINER_REGISTRYCONFIGURATIONS_DEFAULT_SECRET=${METADATA_DEFAULT_DOCKERHUB_PASSWORD}
    depends_on:
      - skipper-server
    restart: always
    volumes:
      - ${DATAFLOW_HOST_MOUNT_PATH:-.}:${DATAFLOW_DOCKER_MOUNT_PATH:-/home/cnb/scdf}
    platform: linux/amd64

  skipper-server:
    image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-2.9.2-SNAPSHOT}${BP_JVM_VERSION:-}
    container_name: skipper
    user: root
    ports:
      - "7577:7577"
      - ${APPS_PORT_RANGE:-20000-20195:20000-20195}
    environment:
      - LANG=en_US.utf8
      - LC_ALL=en_US.utf8
      - JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
      - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_LOW=20000
      - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_HIGH=20190
      - LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_CLOUD_SKIPPER_SERVER_DEPLOYER=ERROR
    volumes:
      - ${DATAFLOW_HOST_MOUNT_PATH:-.}:${DATAFLOW_DOCKER_MOUNT_PATH:-/home/cnb/scdf}
    platform: linux/amd64
    restart: always

  app-import-stream:
    image: springcloud/baseimage:${BASEIMAGE_VERSION:-1.0.1}
    container_name: dataflow-app-import-stream
    depends_on:
      - dataflow-server
    platform: linux/amd64

  app-import-task:
    image: springcloud/baseimage:${BASEIMAGE_VERSION:-1.0.1}
    container_name: dataflow-app-import-task
    depends_on:
      - dataflow-server
    command: >
      /bin/sh -c "
        ./wait-for-it.sh -t 360 dataflow-server:9393;
        wget -qO- '${DATAFLOW_URI:-http://dataflow-server:9393}/apps' --no-check-certificate --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-maven-latest&force=true}';
        echo 'Maven Task apps imported'"
    platform: linux/amd64