Merge pull request #280 from communitiesuk/bau/migrate-with-paketo-image

Run DB migrations with paketo

.github/workflows/copilot_deploy.yml

@@ -61,7 +61,7 @@ jobs:
     with:
       environment: dev
       app_name: account-store
-      db_name: fsd-account-store
+      run_db_migrations: true
       image_location: ${{ needs.paketo_build.outputs.image_location }}
 
   post_dev_deploy_tests:
@@ -92,7 +92,7 @@ jobs:
     with:
       environment: test
       app_name: account-store
-      db_name: fsd-account-store
+      run_db_migrations: true
       image_location: ${{ needs.paketo_build.outputs.image_location }}
 
   post_test_deploy_tests:
@@ -124,7 +124,7 @@ jobs:
     with:
       environment: uat
       app_name: account-store
-      db_name: fsd-account-store
+      run_db_migrations: true
       image_location: ${{ needs.paketo_build.outputs.image_location }}
 
   post_uat_deploy_tests:
@@ -156,5 +156,5 @@ jobs:
     with:
       environment: prod
       app_name: account-store
-      db_name: fsd-account-store
+      run_db_migrations: true
       image_location: ${{ needs.paketo_build.outputs.image_location }}

scripts/migration-task-script.py

@@ -3,36 +3,120 @@
 import subprocess
 import sys
 
-command_to_run = "flask db upgrade"
+
+def get_db_revision_id() -> str | None:
+    """Get the current revision ID of the database
+
+    We are doing `flask db current 2> /dev/null` and then grepping it,
+    because we need the output of the copilot svc exec command to be short.
+
+    Copilot svc exec uses AWS ssm session-manager, which isn't designed to run
+    in github actions, where it doesn't have a TTY.
+
+    Too much output leads to output buffering, which leads to github actions not
+    getting all of the output. Revision ID is on the last line, so it would
+    not show up.
+
+    By getting the command to generate only a few lines, we ensure that the
+    github action reliably gets the revision ID from copilot.
+    """
+
+    try:
+        grep_result = subprocess.check_output(
+            (
+                "copilot svc exec --name fsd-account-store --yes false "
+                "--command \"bash -c 'FLASK_APP=app:application launcher flask db current 2> /dev/null | grep head'\""
+            ),
+            shell=True,
+            text=True,
+        )
+    except subprocess.CalledProcessError as e:
+        sys.exit(e.returncode)
+
+    match = re.search(r"[a-zA-Z0-9_]+(?=\s\(head\))", grep_result)
+
+    return match.group() if match else None
+
+
+def init_task() -> str:
+    try:
+        task_command = subprocess.run(
+            args=[
+                "copilot",
+                "task",
+                "run",
+                "--generate-cmd",
+                f"pre-award/{environment}/fsd-account-store",
+            ],
+            capture_output=True,
+            check=True,
+            text=True,
+        )
+    except subprocess.CalledProcessError as e:
+        sys.exit(e.returncode)
+
+    return task_command.stderr
+
+
+def run_migration_in_task(task_command: str, extra_args: list[str]) -> None:
+    # Remove final line break and append arguments
+    try:
+        subprocess.run(
+            args=task_command[:-1] + " \\\n" + " \\\n".join(extra_args),
+            shell=True,
+            check=True,
+        )
+    except subprocess.CalledProcessError as e:
+        # Don't want to leak the command output here so just exit with the command's return code
+        sys.exit(e.returncode)
+
+
+def get_current_alembic_head_file_content() -> str:
+    """Get the content of .current-alembic-head"""
+
+    with open("db/migrations/.current-alembic-head", "r") as file:
+        file_content = file.read().strip()
+
+    return file_content
+
+
+def has_migrations_to_apply() -> bool:
+    """Check if there are migrations to apply
+
+    If `flask db current` and `.current-alembic-head` have the same revision ID,
+    there are no migrations to apply.
+    """
+
+    current_revision_id = get_db_revision_id()
+    print(f"`flask db current` revision ID: '{current_revision_id}'")
+
+    current_alembic_head = get_current_alembic_head_file_content()
+    print(f"`.current-alembic-head` revision ID: '{current_alembic_head}'")
+
+    return current_revision_id != current_alembic_head
+
+
+task_memory = 2048
 environment = sys.argv[1]
+docker_image = sys.argv[2]
+
+# Check if there are migrations to apply
+if not has_migrations_to_apply():
+    print("No migrations to apply")
+    sys.exit(0)
+
+# Apply migrations
+extra_args = [
+    "--follow",
+    f"--memory {task_memory}",
+    "--entrypoint launcher",
+    "--command 'flask db upgrade'",
+    f"--image {docker_image}",
+]
+
+task_command = init_task()
+
+run_migration_in_task(task_command, extra_args)
 
-try:
-    command = subprocess.run(
-        args=[
-            "copilot",
-            "task",
-            "run",
-            "--generate-cmd",
-            f"pre-award/{environment}/fsd-account-store",
-        ],
-        capture_output=True,
-        check=True,
-        text=True,
-    )
-    # Strip image argument as we want to build a new image to pick up new migrations
-    command_with_image_removed = re.sub(r"--image \S+", "", command.stderr)
-except subprocess.CalledProcessError as e:
-    print(e.stderr)
-    raise e
-
-# Remove final line break and append arguments
-try:
-    subprocess.run(
-        args=command_with_image_removed[:-1]
-        + f" \\\n--follow \\\n--command '{command_to_run}'",
-        shell=True,
-        check=True,
-    )
-except subprocess.CalledProcessError as e:
-    # Don't want to leak the command output here so just exit with the command's return code
-    sys.exit(e.returncode)
+print("Migrations applied")
+sys.exit(0)