Revert back to two Docker Compose files (to simplify PR code review)

microbiomedata · Mar 6, 2025 · 5017b26 · 5017b26
1 parent 332dd7a
commit 5017b26
Show file tree

Hide file tree

Showing 3 changed files with 186 additions and 127 deletions.
diff --git a/docker-compose.base.yml b/docker-compose.base.yml
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -1,34 +1,120 @@
 services:
 
   # Postgres server used by Dagster.
-  # Reference: https://docs.docker.com/reference/compose-file/services/#extends
+  #
+  # > This service runs the postgres DB used by dagster for run storage, schedule storage,
+  # > and event log storage.
+  # > Tests use `postgres:11` image.
+  # > https://github.com/dagster-io/dagster/blob/0.11.9/python_modules/libraries/dagster-postgres/dagster_postgres_tests/docker-compose.yml
+  #
   dagster-postgresql:
-    extends: { file: docker-compose.base.yml, service: dagster-postgresql }
+    image: postgres:11
+    container_name: dagster-postgresql
+    environment:
+      POSTGRES_USER: "postgres_user"
+      POSTGRES_PASSWORD: "postgres_password"
+      POSTGRES_DB: "postgres_db"
     volumes:
       - nmdc_runtime_test_postgres_data:/var/lib/postgresql/data
 
   # Web server that hosts the Dagster web UI.
+  #
+  # > This service runs dagit.
+  # > Since our instance uses the QueuedRunCoordinator, any runs submitted from dagit will be put on
+  # > a queue and later dequeued and launched by dagster-daemon.
+  #
   dagster-dagit:
-    extends: { file: docker-compose.base.yml, service: dagster-dagit }
+    build:
+      context: .
+      dockerfile: nmdc_runtime/Dockerfile
+      target: dagster
+    container_name: dagster-dagit
+    entrypoint: ["tini", "--", "../lib/nmdc_runtime/site/entrypoint-dagit.sh"]
+    expose:
+      - "3000"
+    ports:
+      - "3000:3000"
+    environment:
+      DAGSTER_POSTGRES_USER: "postgres_user"
+      DAGSTER_POSTGRES_PASSWORD: "postgres_password"
+      DAGSTER_POSTGRES_DB: "postgres_db"
+    depends_on:
+      - dagster-postgresql
+    restart: on-failure
+    volumes:
+      - ./:/opt/dagster/lib
     env_file: .env.test
 
   # Dagster daemon.
+  #
+  # > This service runs the dagster-daemon process, which is responsible for taking runs
+  # > off of the queue and launching them, as well as creating runs from schedules or sensors.
+  #
   dagster-daemon:
-    extends: { file: docker-compose.base.yml, service: dagster-daemon }
+    build:
+      context: .
+      dockerfile: nmdc_runtime/Dockerfile
+      target: dagster
+    container_name: dagster-daemon
+    entrypoint: ["tini", "--", "../lib/nmdc_runtime/site/entrypoint-daemon.sh"]
+    restart: on-failure
+    environment:
+      DAGSTER_POSTGRES_USER: "postgres_user"
+      DAGSTER_POSTGRES_PASSWORD: "postgres_password"
+      DAGSTER_POSTGRES_DB: "postgres_db"
+    depends_on:
+      dagster-postgresql: { condition: service_started }
+      # Wait until the MongoDB replica set has been set up by the "init container"
+      # before starting this Dagster daemon container.
+      mongo-init: { condition: service_completed_successfully }
+    volumes:
+      - ./:/opt/dagster/lib
     env_file: .env.test
 
   # Uvicorn server hosting the FastAPI application.
   fastapi:
-    extends: { file: docker-compose.base.yml, service: fastapi }
+    build:
+      context: .
+      dockerfile: nmdc_runtime/Dockerfile
+      target: fastapi
+    container_name: fastapi
+    ports:
+      - "8000:8000"
+    volumes:
+      - .:/code
+    # Wait until the MongoDB replica set has been set up by the "init container"
+    # before starting this FastAPI container.
+    depends_on:
+      mongo-init: { condition: service_completed_successfully }
     env_file: .env.test
 
   # Short-lived MongoDB server used to initialize the one used by the FastAPI application.
   mongo-init:
-    extends: { file: docker-compose.base.yml, service: mongo-init }
+    image: mongo:8.0.4
+    container_name: mongo-init
+    volumes:
+      - ./wait-for-it.sh:/wait-for-it.sh:ro
+      - .docker/mongo_init/initialize_replica_set.sh:/initialize_replica_set.sh:ro
+    depends_on:
+      mongo: { condition: service_started }
+    entrypoint: /bin/bash /wait-for-it.sh --host=mongo --port=27017 --timeout=20 -- /bin/sh /initialize_replica_set.sh mongo 27017 admin root
 
   # MongoDB server used by the FastAPI application.
   mongo:
-    extends: { file: docker-compose.base.yml, service: mongo }
+    image: mongo:8.0.4
+    container_name: mongo
+    ports:
+      - "27018:27017"
+    restart: unless-stopped
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: admin
+      MONGO_INITDB_ROOT_PASSWORD: root
+    # Configure MongoDB to run in replica set mode, so we can use MongoDB transactions.
+    #
+    # Note: Including a KeyFile is necessary when doing the combination of (a) running MongoDB in
+    #       replica set mode and (b) running MongoDB with authentication enabled.
+    #
+    command: ["--replSet", "rs0", "--bind_ip_all", "--keyFile", "/keyFile"]
     volumes:
       - nmdc_runtime_test_mongo_data:/data/db
       - ./mongoKeyFile:/keyFile:ro

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,36 +1,122 @@
 services:
 
   # Postgres server used by Dagster.
-  # Reference: https://docs.docker.com/reference/compose-file/services/#extends
+  #
+  # > This service runs the postgres DB used by dagster for run storage, schedule storage,
+  # > and event log storage.
+  # > Tests use `postgres:11` image.
+  # > https://github.com/dagster-io/dagster/blob/0.11.9/python_modules/libraries/dagster-postgres/dagster_postgres_tests/docker-compose.yml
+  #
   dagster-postgresql:
-    extends: { file: docker-compose.base.yml, service: dagster-postgresql }
+    image: postgres:11
+    container_name: dagster-postgresql
+    environment:
+      POSTGRES_USER: "postgres_user"
+      POSTGRES_PASSWORD: "postgres_password"
+      POSTGRES_DB: "postgres_db"
     volumes:
       - nmdc_runtime_postgres_data:/var/lib/postgresql/data
 
   # Web server that hosts the Dagster web UI.
+  #
+  # > This service runs dagit.
+  # > Since our instance uses the QueuedRunCoordinator, any runs submitted from dagit will be put on
+  # > a queue and later dequeued and launched by dagster-daemon.
+  #
   dagster-dagit:
-    extends: { file: docker-compose.base.yml, service: dagster-dagit }
+    build:
+      context: .
+      dockerfile: nmdc_runtime/Dockerfile
+      target: dagster
+    container_name: dagster-dagit
+    entrypoint: ["tini", "--", "../lib/nmdc_runtime/site/entrypoint-dagit.sh"]
+    expose:
+      - "3000"
+    ports:
+      - "3000:3000"
+    environment:
+      DAGSTER_POSTGRES_USER: "postgres_user"
+      DAGSTER_POSTGRES_PASSWORD: "postgres_password"
+      DAGSTER_POSTGRES_DB: "postgres_db"
+    depends_on:
+      - dagster-postgresql
+    restart: on-failure
+    volumes:
+      - ./:/opt/dagster/lib
     env_file: .env
 
   # Dagster daemon.
+  #
+  # > This service runs the dagster-daemon process, which is responsible for taking runs
+  # > off of the queue and launching them, as well as creating runs from schedules or sensors.
+  #
   dagster-daemon:
-    extends: { file: docker-compose.base.yml, service: dagster-daemon }
+    build:
+      context: .
+      dockerfile: nmdc_runtime/Dockerfile
+      target: dagster
+    container_name: dagster-daemon
+    entrypoint: ["tini", "--", "../lib/nmdc_runtime/site/entrypoint-daemon.sh"]
+    restart: on-failure
+    environment:
+      DAGSTER_POSTGRES_USER: "postgres_user"
+      DAGSTER_POSTGRES_PASSWORD: "postgres_password"
+      DAGSTER_POSTGRES_DB: "postgres_db"
+    depends_on:
+      dagster-postgresql: { condition: service_started }
+      # Wait until the MongoDB replica set has been set up by the "init container"
+      # before starting this Dagster daemon container.
+      mongo-init: { condition: service_completed_successfully }
+    volumes:
+      - ./:/opt/dagster/lib
     env_file: .env
 
   # Uvicorn server hosting the FastAPI application.
   fastapi:
-    extends: { file: docker-compose.base.yml, service: fastapi }
+    build:
+      context: .
+      dockerfile: nmdc_runtime/Dockerfile
+      target: fastapi
+    container_name: fastapi
+    ports:
+      - "8000:8000"
+    volumes:
+      - .:/code
+    # Wait until the MongoDB replica set has been set up by the "init container"
+    # before starting this FastAPI container.
+    depends_on:
+      mongo-init: { condition: service_completed_successfully }
     command: ["uvicorn", "nmdc_runtime.api.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000"]
     env_file: .env
 
   # Short-lived MongoDB server used to initialize the one used by the FastAPI application.
   mongo-init:
-    extends: { file: docker-compose.base.yml, service: mongo-init }
+    image: mongo:8.0.4
+    container_name: mongo-init
+    volumes:
+      - ./wait-for-it.sh:/wait-for-it.sh:ro
+      - .docker/mongo_init/initialize_replica_set.sh:/initialize_replica_set.sh:ro
+    depends_on:
+      mongo: { condition: service_started }
+    entrypoint: /bin/bash /wait-for-it.sh --host=mongo --port=27017 --timeout=20 -- /bin/sh /initialize_replica_set.sh mongo 27017 admin root
 
   # MongoDB server used by the FastAPI application.
   # TODO: Why does this service, which is in the dev—not test—stack, mount a volume named `tests` — is that intentional?
   mongo:
-    extends: { file: docker-compose.base.yml, service: mongo }
+    image: mongo:8.0.4
+    container_name: mongo
+    ports:
+      - "27018:27017"
+    restart: unless-stopped
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: admin
+      MONGO_INITDB_ROOT_PASSWORD: root
+    # Configure MongoDB to run in replica set mode, so we can use MongoDB transactions.
+    #
+    # Note: Including a KeyFile is necessary when doing the combination of (a) running MongoDB in
+    #       replica set mode and (b) running MongoDB with authentication enabled.
+    #
+    command: ["--replSet", "rs0", "--bind_ip_all", "--keyFile", "/keyFile"]
     volumes:
       - nmdc_runtime_mongo_data:/data/db
       - ./mongoKeyFile:/keyFile:ro