Merge pull request #14 from moneymeets/feature/MD-7480-speed-up-ecs-a…

…pplication-deployments

feat(action): run migration task before updating service

.github/workflows/ci.yml

@@ -13,4 +13,4 @@ jobs:
 
       - uses: moneymeets/moneymeets-composite-actions/lint-python@master
 
-      - run: poetry run python -m pytest --cov --cov-fail-under=94
+      - run: poetry run python -m pytest --cov --cov-fail-under=93

action.yml

@@ -79,6 +79,72 @@ runs:
         aws_secret_access_key: ${{ inputs.aws_secret_access_key }}
         aws_region: ${{ inputs.aws_region }}
 
+    - name: Render and deploy preflight task definition to Amazon ECS
+      id: deploy-preflight-task-definition
+      uses: moneymeets/action-ecs-deploy/custom-deploy-steps/create-task-definition@master
+      with:
+        working_directory: ${{ github.action_path }}
+        application_id: ${{ inputs.ecr_repository }}-preflight-${{ inputs.environment }}
+        image_uri: ${{ steps.get-image-uri.outputs.image-uri }}
+        deployment_tag: ${{ inputs.deployment_tag }}
+        aws_access_key_id: ${{ inputs.aws_access_key_id }}
+        aws_secret_access_key: ${{ inputs.aws_secret_access_key }}
+        aws_region: ${{ inputs.aws_region }}
+
+    - name: Run preflight
+      id: run-preflight
+      shell: bash
+      run: |
+        NETWORK_CONFIG=$(
+          aws ecs describe-services \
+            --cluster ${{ inputs.environment }} \
+            --services ${{ inputs.ecr_repository }}-${{ inputs.environment }} \
+            | jq -r '.services[0] | .networkConfiguration.awsvpcConfiguration'
+        )
+
+        SUBNET=$(echo $NETWORK_CONFIG | jq -r '.subnets')
+        SECURITY_GROUPS=$(echo $NETWORK_CONFIG | jq -r '.securityGroups')
+
+        PREFLIGHT_TASK_ARN=$(
+          aws ecs run-task \
+            --cluster ${{ inputs.environment }} \
+            --count 1 \
+            --launch-type "FARGATE" \
+            --network-configuration "{ \"awsvpcConfiguration\": { \"subnets\": $SUBNET, \"securityGroups\": $SECURITY_GROUPS, \"assignPublicIp\": \"DISABLED\" } }" \
+            --task-definition "${{ steps.deploy-preflight-task-definition.outputs.latest-task-definition-arn }}" \
+            | jq -r '.tasks[0] | .taskArn'
+        )
+
+        echo "preflight-task-arn=$PREFLIGHT_TASK_ARN" >> $GITHUB_OUTPUT
+
+    - name: Wait for preflight to finish
+      shell: bash
+      working-directory: ${{ github.action_path }}
+      run: |
+        poetry run actions_helper wait-for-task-stopped \
+          --cluster ${{ inputs.environment }} \
+          --task "${{ steps.run-preflight.outputs.preflight-task-arn }}"
+
+    - name: Determine preflight task state
+      shell: bash
+      working-directory: ${{ github.action_path }}
+      run: |
+        PREFLIGHT_TASK_STATE=$(
+           aws ecs describe-tasks \
+              --cluster ${{ inputs.environment }} \
+              --tasks "${{ steps.run-preflight.outputs.preflight-task-arn }}"
+         )
+        
+        STOPPED_PREFLIGHT_CONTAINER=$(echo $PREFLIGHT_TASK_STATE | jq -r '.tasks[0].containers[0]')
+        EXIT_CODE=$(echo $STOPPED_PREFLIGHT_CONTAINER | jq -r '.exitCode')
+        REASON=$(echo $STOPPED_PREFLIGHT_CONTAINER | jq -r '.reason')
+          
+        if [ "$EXIT_CODE" -ne 0 ]; then
+            echo "Error: Preflight container failed with a non zero exit code - $EXIT_CODE";
+            echo "Reason: $REASON";
+            exit 1;
+        fi
+
     # Service is managed by Pulumi, only desired count and task definition should be updated here
     - name: Update service
       shell: bash
@@ -118,11 +184,13 @@ runs:
               if [ ! -z "$PREVIOUS_PRODUCTION_TASK_DEFINITION" ]; then   
                 echo "TASK_DEFINITION_PRODUCTION=$PREVIOUS_PRODUCTION_TASK_DEFINITION" >> $GITHUB_ENV
                 echo "TASK_DEFINITION_LOCAL=${{ steps.deploy-local-task-definition.outputs.previous-task-definition-arn }}" >> $GITHUB_ENV
+                echo "TASK_DEFINITION_PREFLIGHT=${{ steps.deploy-preflight-task-definition.outputs.previous-task-definition-arn }}" >> $GITHUB_ENV
                 echo "fail-pipeline=false" >> $GITHUB_OUTPUT
               fi
         else
             echo "TASK_DEFINITION_PRODUCTION=$LATEST_PRODUCTION_TASK_DEFINITION" >> $GITHUB_ENV
             echo "TASK_DEFINITION_LOCAL=${{ steps.deploy-local-task-definition.outputs.latest-task-definition-arn }}" >> $GITHUB_ENV
+            echo "TASK_DEFINITION_PREFLIGHT=${{ steps.deploy-preflight-task-definition.outputs.latest-task-definition-arn }}" >> $GITHUB_ENV
             echo "fail-pipeline=true" >> $GITHUB_OUTPUT
         fi
 
@@ -136,6 +204,11 @@ runs:
       shell: bash
       run: aws ecs deregister-task-definition --task-definition "$TASK_DEFINITION_LOCAL"
 
+    - name: Deregister preflight task definition
+      if: ${{ always() && steps.deploy-preflight-task-definition.outcome == 'success' && env.TASK_DEFINITION_PREFLIGHT != '' }}
+      shell: bash
+      run: aws ecs deregister-task-definition --task-definition "$TASK_DEFINITION_PREFLIGHT"
+
     - name: Fail pipeline
       if: ${{ always() && steps.determine-task-definitions-to-deregister.outputs.fail-pipeline == 'true'  }}
       shell: bash

actions_helper/commands/wait_for_service_stability.py

@@ -10,7 +10,7 @@ def wait_for_service_stability(ecs_client: BaseClient, cluster: str, service: st
         services=(service,),
         WaiterConfig={
             # Note: The timeout (= delay * max_attempts) must not be shorter than the workflow timeout!
-            "Delay": 15,  # seconds to wait between retries
+            "Delay": 2,  # seconds to wait between retries
             "MaxAttempts": 1440,
         },
     )

actions_helper/commands/wait_for_task_stopped.py

@@ -0,0 +1,16 @@
+from botocore.client import BaseClient
+
+
+def wait_for_task_stopped(ecs_client: BaseClient, cluster: str, task: str):
+    # Using Boto3 instead CLI in order to control delay and max attempts, see
+    # https://docs.aws.amazon.com/cli/latest/reference/ecs/wait/tasks-stopped.html and
+    # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs/waiter/TasksStopped.html
+    ecs_client.get_waiter("tasks_stopped").wait(
+        cluster=cluster,
+        tasks=(task,),
+        WaiterConfig={
+            # Note: The timeout (= delay * max_attempts) must not be shorter than the workflow timeout!
+            "Delay": 2,  # seconds to wait between retries
+            "MaxAttempts": 1440,
+        },
+    )

actions_helper/main.py

@@ -5,6 +5,7 @@
 
 from actions_helper.commands.get_active_task_definition_by_tag import get_active_task_definition_arn_by_tag
 from actions_helper.commands.wait_for_service_stability import wait_for_service_stability
+from actions_helper.commands.wait_for_task_stopped import wait_for_task_stopped
 
 
 @click.group()
@@ -28,6 +29,22 @@ def cmd_wait_for_service_stability(
     )
 
 
+@cli.command(name="wait-for-task-stopped")
+@click.option("--aws-region", default=os.environ.get("AWS_DEFAULT_REGION"), type=str)
+@click.option("--cluster", type=str)
+@click.option("--task", type=str)
+def cmd_wait_for_task_stopped(
+    aws_region: str,
+    cluster: str,
+    task: str,
+):
+    wait_for_task_stopped(
+        ecs_client=boto3.Session(region_name=aws_region).client("ecs"),
+        cluster=cluster,
+        task=task,
+    )
+
+
 @cli.command(
     name="get-active-task-definition-arn-by-tag",
     short_help="Get active task definition ARN  by specified tags",