Add docker_image

diskuv · Oct 3, 2024 · b49d3dd · b49d3dd
1 parent b848316
commit b49d3dd
Show file tree

Hide file tree

Showing 12 changed files with 388 additions and 148 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -2,10 +2,16 @@
 
 ## 2.1.4
 
-- Add `dockcross_assemble_command` environment variable to inject a shell command into the Docker container during its build assembly.
 - bugfix: Update apt and yum while building dockcross container
 - Batch all opam pin operations. Fixes <https://gitlab.com/dkml/distributions/dkml/-/issues/29>
-- Remove the first argument (`/work/.ci/sd4/deescalate`) when `docker_runner` is set in the environment. This is a breaking change for undocumented functionality.
+- Remove the first argument (`/work/.ci/sd4/deescalate`) when `docker_runner` is set in the environment.
+  And `docker_runner` will not be used unless `docker_image` is set (see below).
+  This is a breaking change for undocumented functionality.
+- Add `docker_assemble_command` environment variable to inject a shell command (anything that can go into a `RUN` Docker statement) into the Docker container during its build assembly.
+  - If `dockcross_image` is set, then the tag will be `${docker_registry/}dkml-workflows/dockcross:latest`
+  - If `docker_image` is set, then the tag will be `${docker_registry/}dkml-workflows/docker:latest` and the image id will be saved at `.ci/sd4/docker-image-id`.
+    The default `docker_runner`, which you can change, is `docker run --rm --workdir /work`.
+    A `-v "$(pwd):/work"` will be the first argument for `docker_runner`, the image id will be the second argument, and the command line (which assumes `/work` is mounted) will be the remainder of the arguments.
 
 ## 2.1.3
 

diff --git a/src/scripts/setup-dkml.sh b/src/scripts/setup-dkml.sh
@@ -47,17 +47,17 @@ fi
 
 # -------------------------------------------------------------------
 
+docker_image_id=
 docker_fqin_preusername= # fully qualified image name (hostname[:port]/username/reponame[:tag]), the parts before the username (hostname[:port]/)
 if [ -n "${docker_registry:-}" ]; then
     docker_fqin_preusername="$docker_registry/"
 fi
 
 # Extend dockcross. https://github.com/dockcross/dockcross#how-to-extend-dockcross-images
-dockcross_image_id=
 dockcross_cli_image_args=
 if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then
     echo "Doing docker build"
-    section_begin docker-build "Summary: docker build --quiet --tag ${docker_fqin_preusername}dkml-workflows/dockcross"
+    section_begin dockcross-build "Summary: docker build --quiet --tag ${docker_fqin_preusername}dkml-workflows/dockcross"
 
     install -d .ci/sd4/docker-image
     #   Since GitLab CI limits environment variables to 255 characters, if you need to exceed that there are five (5)
@@ -66,14 +66,31 @@ if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then
         "${dockcross_image:-}" "${docker_fqin_preusername}" \
         "${dockcross_packages_apt:-}" "${dockcross_packages_apt2:-}" "${dockcross_packages_apt3:-}" "${dockcross_packages_apt4:-}" "${dockcross_packages_apt5:-}" \
         "${dockcross_packages_yum:-}" "${dockcross_packages_yum2:-}" "${dockcross_packages_yum3:-}" "${dockcross_packages_yum4:-}" "${dockcross_packages_yum5:-}" \
-        "${dockcross_assemble_command:-true}" \
+        "${docker_assemble_command:-true}" \
         >.ci/sd4/docker-image/Dockerfile
     docker build --quiet --tag "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image
 
-    # Save image id to re-use for all remaining dockcross invocations
+    # Save image id to re-use for all remaining invocations
     docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id
-    dockcross_image_id=$(cat .ci/sd4/docker-image-id)
-    dockcross_cli_image_args="--image $dockcross_image_id"
+    docker_image_id=$(cat .ci/sd4/docker-image-id)
+    dockcross_cli_image_args="--image $docker_image_id"
+
+    section_end dockcross-build
+fi
+
+# Extend docker_image.
+if [ "${in_docker:-}" = "true" ] && [ -n "${docker_image:-}" ]; then
+    echo "Doing docker build"
+    section_begin docker-build "Summary: docker build --quiet --tag ${docker_fqin_preusername}dkml-workflows/docker"
+
+    install -d .ci/sd4/docker-image
+    printf "FROM %s\nRUN %s" "$docker_image" "${docker_assemble_command:-true}" \
+        >.ci/sd4/docker-image/Dockerfile
+    docker build --quiet --tag "${docker_fqin_preusername}dkml-workflows/docker:latest" .ci/sd4/docker-image
+
+    # Save image id to re-use for all remaining invocations
+    docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id
+    docker_image_id=$(cat .ci/sd4/docker-image-id)
 
     section_end docker-build
 fi
@@ -138,10 +155,11 @@ original_opam_root_cacheable=${original_opam_root_cacheable}
 unix_opam_root=${unix_opam_root}
 unix_opam_root_cacheable=${unix_opam_root_cacheable}
 docker_registry=${docker_registry:-}
+in_docker=${in_docker:-}
 dockcross_image=${dockcross_image:-}
 dockcross_run_extra_args=${dockcross_run_extra_args:-}
+docker_image=${docker_image:-}
 docker_runner=${docker_runner:-}
-in_docker=${in_docker:-}
 ocaml_options=${ocaml_options:-}
 .
 ----
@@ -207,7 +225,7 @@ do_get_dockcross() {
         section_begin get-dockcross 'Get dockcross binary (ManyLinux)'
         install -d .ci/sd4
         #   shellcheck disable=SC2086
-        docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen
+        docker run ${dockcross_run_extra_args:-} --rm "${docker_image_id}" >.ci/sd4/dockcross.gen
 
         # PROBLEM 1
         # ---------
@@ -303,7 +321,7 @@ if [ "\$BUILDER_UID" = 0 ] && [ "\$BUILDER_GID" = 0 ]; then
     # Handle: dockcross --args "-v X:Y --platform P" --image "..." --
     # Confer: https://github.com/dockcross/dockcross/blob/96d87416f639af0204bdd42553e4b99315ca8476/imagefiles/dockcross#L97C1-L134
     ARG_ARGS=
-    ARG_IMAGE="${dockcross_image_id}"
+    ARG_IMAGE="${docker_image_id}"
     while [[ \$# != 0 ]]; do
         case \$1 in
             --)
@@ -517,12 +535,12 @@ EOF
         echo '___________________'
         do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr
 
-    elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then
+    elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_image:-}" ]; then
 
         cat >.ci/sd4/run-with-env <<EOF
 #!/bin/sh
 set -euf
-exec ${docker_runner:-} /work/.ci/sd4/run-in-docker "\$@"
+exec ${docker_runner:-docker run --rm --workdir /work} -v '\$PWD:/work' '$docker_image_id' /work/.ci/sd4/run-in-docker "\$@"
 EOF
         chmod +x .ci/sd4/run-with-env
 

diff --git a/test/gh-darwin/pre/action.yml b/test/gh-darwin/pre/action.yml
@@ -406,17 +406,17 @@ runs:
 
         # -------------------------------------------------------------------
 
+        docker_image_id=
         docker_fqin_preusername= # fully qualified image name (hostname[:port]/username/reponame[:tag]), the parts before the username (hostname[:port]/)
         if [ -n "${docker_registry:-}" ]; then
             docker_fqin_preusername="$docker_registry/"
         fi
 
         # Extend dockcross. https://github.com/dockcross/dockcross#how-to-extend-dockcross-images
-        dockcross_image_id=
         dockcross_cli_image_args=
         if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then
             echo "Doing docker build"
-            section_begin docker-build "Summary: docker build --quiet --tag ${docker_fqin_preusername}dkml-workflows/dockcross"
+            section_begin dockcross-build "Summary: docker build --quiet --tag ${docker_fqin_preusername}dkml-workflows/dockcross"
 
             install -d .ci/sd4/docker-image
             #   Since GitLab CI limits environment variables to 255 characters, if you need to exceed that there are five (5)
@@ -425,14 +425,31 @@ runs:
                 "${dockcross_image:-}" "${docker_fqin_preusername}" \
                 "${dockcross_packages_apt:-}" "${dockcross_packages_apt2:-}" "${dockcross_packages_apt3:-}" "${dockcross_packages_apt4:-}" "${dockcross_packages_apt5:-}" \
                 "${dockcross_packages_yum:-}" "${dockcross_packages_yum2:-}" "${dockcross_packages_yum3:-}" "${dockcross_packages_yum4:-}" "${dockcross_packages_yum5:-}" \
-                "${dockcross_assemble_command:-true}" \
+                "${docker_assemble_command:-true}" \
                 >.ci/sd4/docker-image/Dockerfile
             docker build --quiet --tag "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image
 
-            # Save image id to re-use for all remaining dockcross invocations
+            # Save image id to re-use for all remaining invocations
             docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id
-            dockcross_image_id=$(cat .ci/sd4/docker-image-id)
-            dockcross_cli_image_args="--image $dockcross_image_id"
+            docker_image_id=$(cat .ci/sd4/docker-image-id)
+            dockcross_cli_image_args="--image $docker_image_id"
+
+            section_end dockcross-build
+        fi
+
+        # Extend docker_image.
+        if [ "${in_docker:-}" = "true" ] && [ -n "${docker_image:-}" ]; then
+            echo "Doing docker build"
+            section_begin docker-build "Summary: docker build --quiet --tag ${docker_fqin_preusername}dkml-workflows/docker"
+
+            install -d .ci/sd4/docker-image
+            printf "FROM %s\nRUN %s" "$docker_image" "${docker_assemble_command:-true}" \
+                >.ci/sd4/docker-image/Dockerfile
+            docker build --quiet --tag "${docker_fqin_preusername}dkml-workflows/docker:latest" .ci/sd4/docker-image
+
+            # Save image id to re-use for all remaining invocations
+            docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id
+            docker_image_id=$(cat .ci/sd4/docker-image-id)
 
             section_end docker-build
         fi
@@ -497,10 +514,11 @@ runs:
         unix_opam_root=${unix_opam_root}
         unix_opam_root_cacheable=${unix_opam_root_cacheable}
         docker_registry=${docker_registry:-}
+        in_docker=${in_docker:-}
         dockcross_image=${dockcross_image:-}
         dockcross_run_extra_args=${dockcross_run_extra_args:-}
+        docker_image=${docker_image:-}
         docker_runner=${docker_runner:-}
-        in_docker=${in_docker:-}
         ocaml_options=${ocaml_options:-}
         .
         ----
@@ -566,7 +584,7 @@ runs:
                 section_begin get-dockcross 'Get dockcross binary (ManyLinux)'
                 install -d .ci/sd4
                 #   shellcheck disable=SC2086
-                docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen
+                docker run ${dockcross_run_extra_args:-} --rm "${docker_image_id}" >.ci/sd4/dockcross.gen
 
                 # PROBLEM 1
                 # ---------
@@ -662,7 +680,7 @@ runs:
             # Handle: dockcross --args "-v X:Y --platform P" --image "..." --
             # Confer: https://github.com/dockcross/dockcross/blob/96d87416f639af0204bdd42553e4b99315ca8476/imagefiles/dockcross#L97C1-L134
             ARG_ARGS=
-            ARG_IMAGE="${dockcross_image_id}"
+            ARG_IMAGE="${docker_image_id}"
             while [[ \$# != 0 ]]; do
                 case \$1 in
                     --)
@@ -876,12 +894,12 @@ runs:
                 echo '___________________'
                 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr
 
-            elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then
+            elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_image:-}" ]; then
 
                 cat >.ci/sd4/run-with-env <<EOF
         #!/bin/sh
         set -euf
-        exec ${docker_runner:-} /work/.ci/sd4/deescalate /work/.ci/sd4/run-in-docker "\$@"
+        exec ${docker_runner:-docker run --rm --workdir /work} -v '\$PWD:/work' '$docker_image_id' /work/.ci/sd4/run-in-docker "\$@"
         EOF
                 chmod +x .ci/sd4/run-with-env
 
@@ -1787,7 +1805,7 @@ runs:
       shell: bash
       run: |
         set -x
-        echo '4b9688-${{ steps.full_matrix_vars.outputs.dkml_host_abi }}-${{ steps.full_matrix_vars.outputs.opam_abi }}' > .ci/sd4/cachekey.opam.binaries
+        echo '16f924-${{ steps.full_matrix_vars.outputs.dkml_host_abi }}-${{ steps.full_matrix_vars.outputs.opam_abi }}' > .ci/sd4/cachekey.opam.binaries
         echo '${{ steps.full_matrix_vars.outputs.abi_pattern }}-${{ steps.full_matrix_vars.outputs.vsstudio_arch }}-${{ steps.full_matrix_vars.outputs.vsstudio_hostarch }}-${{ steps.full_matrix_vars.outputs.vsstudio_dir }}-${{ steps.full_matrix_vars.outputs.vsstudio_vcvarsver }}-${{ steps.full_matrix_vars.outputs.vsstudio_winsdkver }}-${{ steps.full_matrix_vars.outputs.vsstudio_msvspreference }}-${{ steps.full_matrix_vars.outputs.vsstudio_cmakegenerator }}' > .ci/sd4/cachekey.vsstudio
         echo '${{ inputs.OCAML_COMPILER }}-${{ inputs.DISKUV_OPAM_REPOSITORY }}-${{ inputs.DKML_COMPILER }}-${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}' > .ci/sd4/cachekey.ci.inputs
 

diff --git a/test/gh-linux/pre/action.yml b/test/gh-linux/pre/action.yml
@@ -431,17 +431,17 @@ runs:
 
         # -------------------------------------------------------------------
 
+        docker_image_id=
         docker_fqin_preusername= # fully qualified image name (hostname[:port]/username/reponame[:tag]), the parts before the username (hostname[:port]/)
         if [ -n "${docker_registry:-}" ]; then
             docker_fqin_preusername="$docker_registry/"
         fi
 
         # Extend dockcross. https://github.com/dockcross/dockcross#how-to-extend-dockcross-images
-        dockcross_image_id=
         dockcross_cli_image_args=
         if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then
             echo "Doing docker build"
-            section_begin docker-build "Summary: docker build --quiet --tag ${docker_fqin_preusername}dkml-workflows/dockcross"
+            section_begin dockcross-build "Summary: docker build --quiet --tag ${docker_fqin_preusername}dkml-workflows/dockcross"
 
             install -d .ci/sd4/docker-image
             #   Since GitLab CI limits environment variables to 255 characters, if you need to exceed that there are five (5)
@@ -450,14 +450,31 @@ runs:
                 "${dockcross_image:-}" "${docker_fqin_preusername}" \
                 "${dockcross_packages_apt:-}" "${dockcross_packages_apt2:-}" "${dockcross_packages_apt3:-}" "${dockcross_packages_apt4:-}" "${dockcross_packages_apt5:-}" \
                 "${dockcross_packages_yum:-}" "${dockcross_packages_yum2:-}" "${dockcross_packages_yum3:-}" "${dockcross_packages_yum4:-}" "${dockcross_packages_yum5:-}" \
-                "${dockcross_assemble_command:-true}" \
+                "${docker_assemble_command:-true}" \
                 >.ci/sd4/docker-image/Dockerfile
             docker build --quiet --tag "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image
 
-            # Save image id to re-use for all remaining dockcross invocations
+            # Save image id to re-use for all remaining invocations
             docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id
-            dockcross_image_id=$(cat .ci/sd4/docker-image-id)
-            dockcross_cli_image_args="--image $dockcross_image_id"
+            docker_image_id=$(cat .ci/sd4/docker-image-id)
+            dockcross_cli_image_args="--image $docker_image_id"
+
+            section_end dockcross-build
+        fi
+
+        # Extend docker_image.
+        if [ "${in_docker:-}" = "true" ] && [ -n "${docker_image:-}" ]; then
+            echo "Doing docker build"
+            section_begin docker-build "Summary: docker build --quiet --tag ${docker_fqin_preusername}dkml-workflows/docker"
+
+            install -d .ci/sd4/docker-image
+            printf "FROM %s\nRUN %s" "$docker_image" "${docker_assemble_command:-true}" \
+                >.ci/sd4/docker-image/Dockerfile
+            docker build --quiet --tag "${docker_fqin_preusername}dkml-workflows/docker:latest" .ci/sd4/docker-image
+
+            # Save image id to re-use for all remaining invocations
+            docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id
+            docker_image_id=$(cat .ci/sd4/docker-image-id)
 
             section_end docker-build
         fi
@@ -522,10 +539,11 @@ runs:
         unix_opam_root=${unix_opam_root}
         unix_opam_root_cacheable=${unix_opam_root_cacheable}
         docker_registry=${docker_registry:-}
+        in_docker=${in_docker:-}
         dockcross_image=${dockcross_image:-}
         dockcross_run_extra_args=${dockcross_run_extra_args:-}
+        docker_image=${docker_image:-}
         docker_runner=${docker_runner:-}
-        in_docker=${in_docker:-}
         ocaml_options=${ocaml_options:-}
         .
         ----
@@ -591,7 +609,7 @@ runs:
                 section_begin get-dockcross 'Get dockcross binary (ManyLinux)'
                 install -d .ci/sd4
                 #   shellcheck disable=SC2086
-                docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen
+                docker run ${dockcross_run_extra_args:-} --rm "${docker_image_id}" >.ci/sd4/dockcross.gen
 
                 # PROBLEM 1
                 # ---------
@@ -687,7 +705,7 @@ runs:
             # Handle: dockcross --args "-v X:Y --platform P" --image "..." --
             # Confer: https://github.com/dockcross/dockcross/blob/96d87416f639af0204bdd42553e4b99315ca8476/imagefiles/dockcross#L97C1-L134
             ARG_ARGS=
-            ARG_IMAGE="${dockcross_image_id}"
+            ARG_IMAGE="${docker_image_id}"
             while [[ \$# != 0 ]]; do
                 case \$1 in
                     --)
@@ -901,12 +919,12 @@ runs:
                 echo '___________________'
                 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr
 
-            elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then
+            elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_image:-}" ]; then
 
                 cat >.ci/sd4/run-with-env <<EOF
         #!/bin/sh
         set -euf
-        exec ${docker_runner:-} /work/.ci/sd4/deescalate /work/.ci/sd4/run-in-docker "\$@"
+        exec ${docker_runner:-docker run --rm --workdir /work} -v '\$PWD:/work' '$docker_image_id' /work/.ci/sd4/run-in-docker "\$@"
         EOF
                 chmod +x .ci/sd4/run-with-env
 
@@ -1812,7 +1830,7 @@ runs:
       shell: bash
       run: |
         set -x
-        echo '4b9688-${{ steps.full_matrix_vars.outputs.dkml_host_abi }}-${{ steps.full_matrix_vars.outputs.opam_abi }}' > .ci/sd4/cachekey.opam.binaries
+        echo '16f924-${{ steps.full_matrix_vars.outputs.dkml_host_abi }}-${{ steps.full_matrix_vars.outputs.opam_abi }}' > .ci/sd4/cachekey.opam.binaries
         echo '${{ steps.full_matrix_vars.outputs.abi_pattern }}-${{ steps.full_matrix_vars.outputs.vsstudio_arch }}-${{ steps.full_matrix_vars.outputs.vsstudio_hostarch }}-${{ steps.full_matrix_vars.outputs.vsstudio_dir }}-${{ steps.full_matrix_vars.outputs.vsstudio_vcvarsver }}-${{ steps.full_matrix_vars.outputs.vsstudio_winsdkver }}-${{ steps.full_matrix_vars.outputs.vsstudio_msvspreference }}-${{ steps.full_matrix_vars.outputs.vsstudio_cmakegenerator }}' > .ci/sd4/cachekey.vsstudio
         echo '${{ inputs.OCAML_COMPILER }}-${{ inputs.DISKUV_OPAM_REPOSITORY }}-${{ inputs.DKML_COMPILER }}-${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}' > .ci/sd4/cachekey.ci.inputs