Merge pull request #89 from cyber-dojo/make-main-workflow-reusable

Make main workflow reuseable
cyber-dojo · Jan 26, 2025 · 4cb1342 · 4cb1342
2 parents 9d16767 + ed32612
commit 4cb1342
Show file tree

Hide file tree

Showing 9 changed files with 142 additions and 52 deletions.
diff --git a/.github/workflows/base_image_trigger.yml b/.github/workflows/base_image_trigger.yml
@@ -0,0 +1,34 @@
+name: Base Image Update
+
+on:
+  workflow_dispatch:
+    inputs:
+      base_image:
+        description: "Image-Tag for cyberdojo/docker-base (short commit SHA - first 7 digits) eg edb2887"
+        required: true
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      base_image:  ${{ steps.vars.outputs.base_image }}
+      kosli_trail: ${{ steps.vars.outputs.kosli_trail }}
+    steps:
+      - name: Outputs
+        id: vars
+        run: | 
+          echo "base_image=cyberdojo/docker-base:${{ inputs.base_image }}" >> ${GITHUB_OUTPUT}
+          echo "kosli_trail=base-image-update-${{ inputs.base_image }}"    >> ${GITHUB_OUTPUT}
+
+  trigger:
+    needs: [setup]
+    uses: ./.github/workflows/main.yml
+    with:
+      BASE_IMAGE:  ${{ needs.setup.outputs.base_image }}
+      KOSLI_TRAIL: ${{ needs.setup.outputs.kosli_trail }}
+    secrets:
+      DOCKER_USER:     ${{ secrets.DOCKER_USER }}
+      DOCKER_PASS:     ${{ secrets.DOCKER_PASS }}
+      SNYK_TOKEN:      ${{ secrets.SNYK_TOKEN }}
+      KOSLI_API_TOKEN: ${{ secrets.KOSLI_API_TOKEN }}
+
diff --git a/.github/workflows/commit_trigger.yml b/.github/workflows/commit_trigger.yml
@@ -0,0 +1,33 @@
+name: Main
+
+on:
+  push:
+
+jobs:
+  get-base-image:
+    runs-on: ubuntu-latest
+    outputs:
+      base_image: ${{ steps.vars.outputs.base_image }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Output the base-image
+        id: vars
+        run: |
+          source ./bin/lib.sh
+          echo "base_image=$(echo_base_image)" >> ${GITHUB_OUTPUT}
+
+  trigger:
+    needs: [get-base-image]
+    uses: ./.github/workflows/main.yml
+    with:
+      BASE_IMAGE:  ${{ needs.get-base-image.outputs.base_image }}
+      KOSLI_TRAIL: ${{ github.sha }}
+    secrets:
+      DOCKER_USER:     ${{ secrets.DOCKER_USER }}
+      DOCKER_PASS:     ${{ secrets.DOCKER_PASS }}
+      SNYK_TOKEN:      ${{ secrets.SNYK_TOKEN }}
+      KOSLI_API_TOKEN: ${{ secrets.KOSLI_API_TOKEN }}
+
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,19 +1,36 @@
-name: Main
+name: Sub - main
 
 on:
-  push:
+  workflow_call:
+    inputs:
+      BASE_IMAGE:
+        required: true
+        type: string
+      KOSLI_TRAIL:
+        required: true
+        type: string
+    secrets:
+      DOCKER_USER:
+        required: true
+      DOCKER_PASS:
+        required: true
+      SNYK_TOKEN:
+        required: true
+      KOSLI_API_TOKEN:
+        required: true
 
 env:
   KOSLI_DRY_RUN: ${{ vars.KOSLI_DRY_RUN }}           # false
   KOSLI_HOST: ${{ vars.KOSLI_HOST }}                 # https://app.kosli.com
+  KOSLI_API_TOKEN: ${{ secrets.KOSLI_API_TOKEN }}
   KOSLI_ORG: ${{ vars.KOSLI_ORG }}                   # cyber-dojo
   KOSLI_FLOW: ${{ vars.KOSLI_FLOW }}                 # runner-ci
-  KOSLI_API_TOKEN: ${{ secrets.KOSLI_API_TOKEN }}
   KOSLI_TRAIL: ${{ github.sha }}
-  SERVICE_NAME: ${{ github.event.repository.name }}  # runner
+
   AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }}
   AWS_ECR_ID: ${{ vars.AWS_ECR_ID }}
   AWS_REGION: ${{ vars.AWS_REGION }}
+  SERVICE_NAME: ${{ github.event.repository.name }}  # runner
   IMAGE_TAR_FILENAME: /tmp/${{ github.event.repository.name }}:${{ github.sha }}.tar
   DOCKER_API_VERSION: ${{ vars.DOCKER_API_VERSION }}
 
@@ -185,12 +202,6 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
 
-      - name: Output the base-image
-        id: vars
-        run: |
-          source ./bin/lib.sh
-          echo "base_image=$(echo_base_image)" >> ${GITHUB_OUTPUT}
-
       - name: Build and push Docker image to ECR
         id: docker_build
         uses: docker/build-push-action@v5
@@ -200,7 +211,7 @@ jobs:
           tags: ${{ env.IMAGE_NAME }}
           build-args: |
             COMMIT_SHA=${{ github.sha }}
-            BASE_IMAGE=${{ steps.vars.outputs.base_image }}
+            BASE_IMAGE=${{ inputs.base_image }}
 
       - name: Tar Docker image
         run: |

diff --git a/bin/build_image.sh b/bin/build_image.sh
@@ -28,7 +28,7 @@ check_args()
       ;;
     'server')
       if [ -n "${CI:-}" ] ; then
-        stderr "In CI workflow - use previous docker/build-push-action@v6 GitHub Action"
+        stderr "In CI workflow - use docker/build-push-action@v6 GitHub Action"
         exit 42
       fi
       ;;
@@ -51,10 +51,19 @@ build_image()
   check_args "$@"
   local -r type="${1}"
   exit_non_zero_unless_installed docker
+  # shellcheck disable=SC2046
   export $(echo_env_vars)
   containers_down
   remove_old_images
 
+  echo
+  echo "Building with --build-args"
+  echo "  COMMIT_SHA=${COMMIT_SHA}"
+  echo "  BASE_IMAGE=${CYBER_DOJO_RUNNER_BASE_IMAGE}"
+  echo "To change this run:"
+  echo "$ COMMIT_SHA=... CYBER_DOJO_RUNNER_BASE_IMAGE=cyberdojo/docker-base:... make image_${type}"
+  echo
+
   docker compose build server
   if [ "${type}" == 'client' ]; then
     docker compose build client
@@ -72,7 +81,7 @@ build_image()
   if [ "${type}" == 'server' ]; then
     # Create latest tag for image build cache
     docker tag "${image_name}" "${CYBER_DOJO_RUNNER_IMAGE}:latest"
-    # Tag image-name for local development where runners name comes from echo-versioner-env-vars
+    # Tag image-name for local development where runners name comes from echo-env-vars
     docker tag "${image_name}" "cyberdojo/runner:latest"
     echo "CYBER_DOJO_RUNNER_SHA=${CYBER_DOJO_RUNNER_SHA}"
     echo "CYBER_DOJO_RUNNER_TAG=${CYBER_DOJO_RUNNER_TAG}"

diff --git a/bin/check_coverage.sh b/bin/check_coverage.sh
@@ -41,6 +41,7 @@ check_args()
 check_coverage()
 {
   check_args "$@"
+  # shellcheck disable=SC2046
   export $(echo_env_vars)
 
   local -r TYPE="${1}"           # {server|client}

diff --git a/bin/demo.sh b/bin/demo.sh
@@ -6,6 +6,7 @@ export ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 readonly DEMO_FILENAME="/tmp/runner_demo.html"
 
 source "${ROOT_DIR}/bin/lib.sh"
+# shellcheck disable=SC2046
 export $(echo_env_vars)
 docker compose --progress=plain up --wait --wait-timeout=10 client
 docker exec -it test_runner_client ruby /runner/demo.rb > "${DEMO_FILENAME}"

diff --git a/bin/lib.sh b/bin/lib.sh
@@ -1,6 +1,44 @@
 #!/usr/bin/env bash
 set -Eeu
 
+echo_base_image()
+{
+  local -r json="$(curl --fail --silent --request GET https://beta.cyber-dojo.org/runner/base_image)"
+  echo "${json}" | jq -r '.base_image'
+  #echo cyberdojo/docker-base:d6830c0
+}
+
+echo_env_vars()
+{
+  # --build-arg ...
+  if [[ ! -v CYBER_DOJO_RUNNER_BASE_IMAGE ]] ; then
+    echo CYBER_DOJO_RUNNER_BASE_IMAGE="$(echo_base_image)"
+  fi
+  if [[ ! -v COMMIT_SHA ]] ; then
+    local -r sha="$(cd "${ROOT_DIR}" && git rev-parse HEAD)"
+    echo COMMIT_SHA="${sha}"
+  fi
+
+  # From versioner ...
+  docker run --rm cyberdojo/versioner
+
+  echo CYBER_DOJO_RUNNER_SHA="${sha}"
+  echo CYBER_DOJO_RUNNER_TAG="${sha:0:7}"
+
+  echo CYBER_DOJO_RUNNER_CLIENT_IMAGE=cyberdojo/runner-client
+  echo CYBER_DOJO_RUNNER_CLIENT_PORT=9999
+
+  echo CYBER_DOJO_RUNNER_CLIENT_USER=nobody
+  echo CYBER_DOJO_RUNNER_SERVER_USER=root
+
+  echo CYBER_DOJO_RUNNER_CLIENT_CONTAINER_NAME=test_runner_client
+  echo CYBER_DOJO_RUNNER_SERVER_CONTAINER_NAME=test_runner_server
+
+  local -r AWS_ACCOUNT_ID=244531986313
+  local -r AWS_REGION=eu-central-1
+  echo CYBER_DOJO_RUNNER_IMAGE="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/runner"
+}
+
 exit_non_zero_unless_installed()
 {
   for dependent in "$@"
@@ -46,44 +84,6 @@ containers_down()
   docker compose down --remove-orphans --volumes
 }
 
-echo_base_image()
-{
-  local -r json="$(curl --fail --silent --request GET https://beta.cyber-dojo.org/runner/base_image)"
-  echo "${json}" | jq -r '.base_image'
-  #echo cyberdojo/docker-base:d6830c0
-}
-
-echo_env_vars()
-{
-  # --build-arg ...
-  if [[ ! -v CYBER_DOJO_RUNNER_BASE_IMAGE ]] ; then
-    echo CYBER_DOJO_RUNNER_BASE_IMAGE="$(echo_base_image)"
-  fi
-  if [[ ! -v COMMIT_SHA ]] ; then
-    local -r sha="$(cd "${ROOT_DIR}" && git rev-parse HEAD)"
-    echo COMMIT_SHA="${sha}"
-  fi
-
-  # From versioner ...
-  docker run --rm cyberdojo/versioner
-
-  echo CYBER_DOJO_RUNNER_SHA="${sha}"
-  echo CYBER_DOJO_RUNNER_TAG="${sha:0:7}"
-
-  echo CYBER_DOJO_RUNNER_CLIENT_IMAGE=cyberdojo/runner-client
-  echo CYBER_DOJO_RUNNER_CLIENT_PORT=9999
-
-  echo CYBER_DOJO_RUNNER_CLIENT_USER=nobody
-  echo CYBER_DOJO_RUNNER_SERVER_USER=root
-
-  echo CYBER_DOJO_RUNNER_CLIENT_CONTAINER_NAME=test_runner_client
-  echo CYBER_DOJO_RUNNER_SERVER_CONTAINER_NAME=test_runner_server
-
-  local -r AWS_ACCOUNT_ID=244531986313
-  local -r AWS_REGION=eu-central-1
-  echo CYBER_DOJO_RUNNER_IMAGE="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/runner"
-}
-
 remove_old_images()
 {
   echo Removing old images

diff --git a/bin/run_tests.sh b/bin/run_tests.sh
@@ -6,7 +6,7 @@ export ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 source "${ROOT_DIR}/bin/lib.sh"
 source "${ROOT_DIR}/bin/create_test_data_manifests_file.sh"
 source "${ROOT_DIR}/bin/setup_dependent_images.sh"
-
+# shellcheck disable=SC2046
 export $(echo_env_vars)
 
 show_help()

diff --git a/bin/snyk_container_scan.sh b/bin/snyk_container_scan.sh
@@ -3,6 +3,7 @@ set -Eeu
 
 export ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 source "${ROOT_DIR}/bin/lib.sh"
+# shellcheck disable=SC2046
 export $(echo_env_vars)
 
 readonly IMAGE_NAME="${CYBER_DOJO_RUNNER_IMAGE}:${CYBER_DOJO_RUNNER_TAG}"