docker-compose.yml

version: "3.6"
volumes:
  shared-workspace:
    name: "hadoop-distributed-file-system"
    driver: local
  rs1:
  rs2:
  rs3:

services:
  jupyterlab:
    image: jupyter/pyspark-notebook:latest
    container_name: jupyterlab
    environment:
      - JUPYTER_ENABLE_LAB=yes
      - GRANT_SUDO=yes

      
    ports:
      - 8888:8888
    volumes:
      - shared-workspace:/opt/workspace
    networks:
      - localnet

  spark-master:
    image: bde2020/spark-master:latest #3.2.0-hadoop3.2
    container_name: spark-master
    ports:
      - 8090:8080
      - 7077:7077
    volumes:
      - shared-workspace:/opt/workspace
    networks:
      - localnet

  spark-worker-1:
    image: bde2020/spark-worker:latest #3.2.0-hadoop3.2
    container_name: spark-worker-1
    environment:
      - SPARK_WORKER_CORES=2
      - SPARK_WORKER_MEMORY=4g
      - SPARK_MASTER=spark://spark-master:7077
    ports:
      - 8081:8081
    volumes:
      - shared-workspace:/opt/workspace
    depends_on:
      - spark-master
    networks:
      - localnet

  spark-worker-2:
    image: bde2020/spark-worker:latest #3.2.0-hadoop3.2
    container_name: spark-worker-2
    environment:
      - SPARK_WORKER_CORES=4
      - SPARK_WORKER_MEMORY=4g
      - SPARK_MASTER=spark://spark-master:7077
    ports:
      - 8082:8081
    volumes:
      - shared-workspace:/opt/workspace
    depends_on:
      - spark-master
    networks:
      - localnet
  
  # MongoDB Replica Set
  mongo1:
    image: "mongo:latest"
    container_name: mongo1
    command: --replSet rs0 --oplogSize 128 --bind_ip 0.0.0.0
    volumes:
      - rs1:/data/db
    networks:
      - localnet
    ports:
      - "27017:27017"
    restart: always

  mongo2:
    image: "mongo:latest"
    container_name: mongo2
    command: --replSet rs0 --oplogSize 128 --bind_ip 0.0.0.0
    volumes:
      - rs2:/data/db
    networks:
      - localnet
    ports:
      - "27018:27017"
    restart: always

  mongo3:
    image: "mongo:latest"
    container_name: mongo3
    command: --replSet rs0 --oplogSize 128 --bind_ip 0.0.0.0
    volumes:
      - rs3:/data/db
    networks:
      - localnet
    ports:
      - "27019:27017"
    restart: always

networks:
    localnet:
        attachable: true