docker-compose.yaml

services:

  router-server:
    build:
      context: .
      dockerfile: src/router-server/router-server.dockerfile
    image: router-server:latest
    shm_size: 8G
    ulimits:
      memlock: -1
      stack: 67108864
    volumes:
      - ./routers/:/model_repository
    command: tritonserver --log-verbose=1 --model-repository=/model_repository
    ports:
      - "8000:8000"
      - "8001:8001"
      - "8002:8002"
    networks:
      - llm-router-network
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: [ "0" ]
              capabilities: [ gpu ]

  app:
    build:
      context: .
      dockerfile: demo/app/app.dockerfile
    image: llm-router-client:app
    working_dir: /app
    ports:
      - 8008:8008
    networks:
      - llm-router-network
    volumes:
      - type: bind
        source: demo/app
        target: /app
    command: python app.py

  router-controller:
    build:
      context: .
      dockerfile: src/router-controller/router-controller.dockerfile
    image: router-controller:latest
    ports:
      - "8084:8084"
    networks:
      - llm-router-network

  prometheus:
    image: prom/prometheus:v2.35.0
    volumes:
      - ./src/prometheus:/etc/prometheus
    ports:
      - "9090:9090"
    networks:
      - llm-router-network
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.enable-lifecycle'

  grafana:
    image: grafana/grafana:9.1.6
    volumes:
      - grafana_data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=secret # Change this password
    ports:
      - "3000:3000"
    networks:
      - llm-router-network
    depends_on:
      - prometheus

  locust:
    build:
      context: .
      dockerfile: demo/loadtest/locust.dockerfile
    image: locust:latest
    ports:
      - "8089:8089"
      - "4000:4000"
    volumes:
      - ./demo/loadtest/:/mnt/locust
    command: -f /mnt/locust/locustfile.py --host http://router-controller:8084
    networks:
      - llm-router-network

  router-builder:
    build:
      dockerfile: customize/router_builder.dockerfile
    shm_size: 8G
    ulimits:
      memlock: -1
      stack: 67108864
    volumes:
      - ./customize:/workspace
      - ./routers/:/model_repository
    command: jupyter lab --no-browser --allow-root --ip=0.0.0.0 --port=9999 --NotebookApp.token "" --NotebookApp.password "" --notebook-dir "/workspace/"
    ports:
      - "9999:9999"
    networks:
      - llm-router-network
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: [ "0" ]
              capabilities: [ gpu ]

volumes:
  grafana_data:


networks:
  llm-router-network:
    driver: bridge