From 97602e3fa2379b93cd1be99d2c152c2b9abb1539 Mon Sep 17 00:00:00 2001
From: Jose Diaz-Gonzalez <email@josediazgonzalez.com>
Date: Thu, 5 Sep 2019 23:49:44 -0400
Subject: [PATCH] fix: use new triggers for image cache management

The previous implementation of docker image cache utilization ended up reverting the codebase to the one matching the last extended app image, so the only way to ensure new code was deployed was to update a file recognized by this plugin.

In the new implementation, we utilize the new builder plugin triggers to handle image caching, and always rebuild at the pre-build-buildpack phase if there exists a dpkg-packages file.
---
 builder-create-dokku-image |  63 +++++++++++++++++++++
 builder-dokku-image        |  27 +++++++++
 internal-functions         | 111 +++++++++++++++++++++++++++++++++++++
 pre-build-buildpack        | 110 +++++-------------------------------
 4 files changed, 214 insertions(+), 97 deletions(-)
 create mode 100755 builder-create-dokku-image
 create mode 100755 builder-dokku-image
 create mode 100755 internal-functions

diff --git a/builder-create-dokku-image b/builder-create-dokku-image
new file mode 100755
index 0000000..4c7bb5c
--- /dev/null
+++ b/builder-create-dokku-image
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -eo pipefail
+[[ $DOKKU_TRACE ]] && set -x
+export DOCKER_BIN=${DOCKER_BIN:="docker"}
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+source "$PLUGIN_AVAILABLE_PATH/apt/internal-functions"
+
+hook-apt-builder-create-dokku-image() {
+  declare BUILDER_TYPE="$1" APP="$2" SOURCECODE_WORK_DIR="$3" DOKKU_IMAGE="$4"
+  local IMAGE="dokku/$APP" DIR=/tmp/apt
+  local COMMAND CONTENT_SHA DOCKER_COMMIT_LABEL_ARGS DOCKER_RUN_LABEL_ARGS
+
+  if [[ -f "$SOURCECODE_WORK_DIR/dpkg-packages" ]]; then
+    dokku_log_info1 "Rebuilding extended app image due to dpkg-packages usage"
+    return
+  fi
+
+  CONTENT_SHA="$(fn-apt-fetch-sha "$SOURCECODE_WORK_DIR")"
+  if [[ -z "$CONTENT_SHA" ]]; then
+    return
+  fi
+
+  local TMP_WORK_DIR=$(mktemp -d "/tmp/dokku-${DOKKU_PID}-${FUNCNAME[0]}.XXXXXX")
+  trap "rm -rf '$TMP_WORK_DIR' >/dev/null" RETURN
+  fn-apt-populate-work-dir "$SOURCECODE_WORK_DIR" "$TMP_WORK_DIR"
+
+  if [[ "$("$DOCKER_BIN" images --quiet "dokku/$APP:$CONTENT_SHA" 2>/dev/null)" != "" ]]; then
+    dokku_log_info1 "Compatible extended app image found, skipping system package installation"
+    fn-clean-extended-app-images "$APP" "dokku/$APP:$CONTENT_SHA"
+    return
+  fi
+
+  dokku_log_info1 "Creating extended app image with custom system packages"
+  pushd "$TMP_WORK_DIR" >/dev/null
+  CID=$(tar -c . | "$DOCKER_BIN" run "${DOCKER_RUN_LABEL_ARGS[@]}" $DOKKU_GLOBAL_RUN_ARGS -i -a stdin "$DOKKU_IMAGE" /bin/bash -c "mkdir -p /tmp/apt && tar -xC /tmp/apt")
+  popd >/dev/null
+  if test "$("$DOCKER_BIN" wait "$CID")" -ne 0; then
+    dokku_log_warn "Failure extracting apt files"
+    return 1
+  fi
+
+  DOCKER_COMMIT_LABEL_ARGS=("--change" "LABEL org.label-schema.schema-version=1.0" "--change" "LABEL org.label-schema.vendor=dokku" "--change" "LABEL com.dokku.app-name=$APP" "--change" "LABEL $DOKKU_CONTAINER_LABEL=")
+  "$DOCKER_BIN" commit "${DOCKER_COMMIT_LABEL_ARGS[@]}" "$CID" "$IMAGE:apt" >/dev/null
+  "$DOCKER_BIN" rm "$CID" &>/dev/null || true
+
+  COMMAND="$(fn-apt-command "$APP" "$DOKKU_IMAGE" "/tmp/apt")"
+  DOCKER_RUN_LABEL_ARGS="--label=com.dokku.app-name=$APP"
+  CID=$("$DOCKER_BIN" run "${DOCKER_RUN_LABEL_ARGS[@]}" $DOKKU_GLOBAL_RUN_ARGS -d "$IMAGE:apt" /bin/bash -e -c "$COMMAND")
+
+  "$DOCKER_BIN" attach "$CID"
+  if test "$("$DOCKER_BIN" wait "$CID")" -ne 0; then
+    dokku_log_warn "Failure installing system packages"
+    return 1
+  fi
+
+  DOCKER_COMMIT_LABEL_ARGS=("--change" "LABEL org.label-schema.schema-version=1.0" "--change" "LABEL org.label-schema.vendor=dokku" "--change" "LABEL com.dokku.app-name=sha-$APP" "--change" "LABEL $DOKKU_CONTAINER_LABEL=")
+  "$DOCKER_BIN" commit "${DOCKER_COMMIT_LABEL_ARGS[@]}" "$CID" "dokku/$APP:$CONTENT_SHA" >/dev/null
+  "$DOCKER_BIN" rm "$CID" &>/dev/null || true
+  "$DOCKER_BIN" rmi "$IMAGE:apt" &>/dev/null || true
+  fn-clean-extended-app-images "$APP" "dokku/$APP:$CONTENT_SHA"
+}
+
+hook-apt-builder-create-dokku-image "$@"
diff --git a/builder-dokku-image b/builder-dokku-image
new file mode 100755
index 0000000..c5d6328
--- /dev/null
+++ b/builder-dokku-image
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -eo pipefail
+[[ $DOKKU_TRACE ]] && set -x
+export DOCKER_BIN=${DOCKER_BIN:="docker"}
+source "$PLUGIN_AVAILABLE_PATH/apt/internal-functions"
+
+hook-apt-builder-dokku-image() {
+  declare BUILDER_TYPE="$1" APP="$2" SOURCECODE_WORK_DIR="$3"
+  local IMAGE="dokku/$APP"
+  local CONTENT_SHA
+
+  if [[ -f "$SOURCECODE_WORK_DIR/dpkg-packages" ]]; then
+    return
+  fi
+
+  CONTENT_SHA="$(fn-apt-fetch-sha "$SOURCECODE_WORK_DIR")"
+  if [[ -z "$CONTENT_SHA" ]]; then
+    return
+  fi
+
+  if [[ "$("$DOCKER_BIN" images --quiet "$IMAGE:$CONTENT_SHA" 2>/dev/null)" != "" ]]; then
+    echo "$IMAGE:$CONTENT_SHA"
+    return
+  fi
+}
+
+hook-apt-builder-dokku-image "$@"
diff --git a/internal-functions b/internal-functions
new file mode 100755
index 0000000..c32e12b
--- /dev/null
+++ b/internal-functions
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+set -eo pipefail
+[[ $DOKKU_TRACE ]] && set -x
+export DOCKER_BIN=${DOCKER_BIN:="docker"}
+source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
+
+fn-clean-extended-app-images() {
+  declare APP="$1" IMAGE="$2"
+  local images
+
+  # remove dangling extended app images
+  "$DOCKER_BIN" rmi $("$DOCKER_BIN" images --format 'dangling=true' --format "label=com.dokku.app-name=sha-$APP" --quiet) &>/dev/null || true
+
+  images="$("$DOCKER_BIN" images --filter "label=com.dokku.app-name=sha-$APP" --quiet)"
+  for image in $images; do
+    if [[ "$("$DOCKER_BIN" inspect --format '{{(index .RepoTags 0)}}' "$image" 2>/dev/null)" != "$IMAGE" ]]; then
+      "$DOCKER_BIN" rmi "$image" &>/dev/null || true
+    fi
+  done
+}
+
+fn-apt-fetch-sha() {
+  declare SOURCECODE_WORK_DIR="$1"
+  local APT_FILES CONTENT INJECT_PACKAGES file
+
+  if [[ -f "$SOURCECODE_WORK_DIR/dpkg-packages" ]]; then
+    return
+  fi
+
+  APT_FILES=('apt-env' 'apt-preferences' 'apt-sources-list' 'apt-repositories' 'apt-debconf' 'apt-packages')
+  for file in "${APT_FILES[@]}"; do
+    if [[ -f "$SOURCECODE_WORK_DIR/$file" ]]; then
+      INJECT_PACKAGES=true
+      local file_contents=$(<$SOURCECODE_WORK_DIR/$file)
+      CONTENT="${CONTENT}\n${file}\n${file_contents}"
+    fi
+  done
+
+  if [[ "$INJECT_PACKAGES" != "true" ]]; then
+    return
+  fi
+
+  echo -n "$(<$PLUGIN_AVAILABLE_PATH/apt/plugin.toml)$CONTENT" | sha256sum | cut -d " " -f 1
+}
+
+fn-apt-populate-work-dir() {
+  declare SOURCECODE_WORK_DIR="$1" TMP_WORK_DIR="$2"
+  local APT_FILES file
+
+  APT_FILES=('apt-env' 'apt-preferences' 'apt-sources-list' 'apt-repositories' 'apt-debconf' 'apt-packages')
+  for file in "${APT_FILES[@]}"; do
+    if [[ -f "$SOURCECODE_WORK_DIR/$file" ]]; then
+      cp "$SOURCECODE_WORK_DIR/$file" "$TMP_WORK_DIR/$file"
+    fi
+  done
+}
+
+fn-apt-command() {
+  declare APP="$1" DOKKU_IMAGE="$2" DIR="$3"
+  cat <<EOF
+# $APP $DOKKU_IMAGE
+sleep 2
+export DEBIAN_FRONTEND=noninteractive
+if [ -f $DIR/apt-env ]; then
+    echo "-----> Sourcing apt env"
+    source $DIR/apt-env
+fi
+if [ -d $DIR/apt-preferences ]; then
+    echo "-----> Injecting apt preferences"
+    mv -v $DIR/apt-preferences /etc/apt/preferences.d/90customizations
+fi
+if [ -f $DIR/apt-sources-list ]; then
+    echo "-----> Using customized sources.list"
+    mv -v $DIR/apt-sources-list /etc/apt/sources.list
+fi
+if [ -f $DIR/apt-repositories ]; then
+    echo "-----> Updating package list"
+    apt-get update >/dev/null
+    echo "-----> Installing required apt transport packages"
+    apt-get install -y software-properties-common apt-transport-https
+    echo "-----> Installing custom apt repositories"
+    cat "$DIR/apt-repositories" | while read repository; do
+        if [ -n "\$repository" ]; then
+            add-apt-repository -y "\$repository"
+        fi
+    done
+fi
+if [ -f $DIR/apt-debconf ]; then
+    cat "$DIR/apt-debconf" | while read conf; do
+        if [ -n "\$conf" ]; then
+            echo \$conf | debconf-set-selections
+        fi
+    done
+fi
+if [ -f $DIR/apt-packages ]; then
+    PACKAGES=\$(cat "$DIR/apt-packages" | tr "\\n" " ")
+    echo "-----> Updating package list"
+    apt-get update >/dev/null
+    echo "-----> Injecting packages: \$PACKAGES"
+    apt-get install -y \$PACKAGES
+fi
+if [ -d $DIR/dpkg-packages ]; then
+    for pkg in $DIR/dpkg-packages/*.deb; do
+        echo "-----> Injecting package: \$pkg"
+        dpkg -i \$pkg
+    done
+fi
+rm -rf /tmp/apt
+sleep 1 # wait so that docker run has not exited before docker attach
+EOF
+}
diff --git a/pre-build-buildpack b/pre-build-buildpack
index 81eef6a..65748ec 100755
--- a/pre-build-buildpack
+++ b/pre-build-buildpack
@@ -3,122 +3,38 @@ set -eo pipefail
 [[ $DOKKU_TRACE ]] && set -x
 export DOCKER_BIN=${DOCKER_BIN:="docker"}
 source "$PLUGIN_CORE_AVAILABLE_PATH/common/functions"
-
-fn-clean-extended-app-images() {
-  declare APP="$1" IMAGE="$2"
-  local images
-
-  # remove dangling extended app images
-  "$DOCKER_BIN" rmi $("$DOCKER_BIN" images --format 'dangling=true' --format "label=com.dokku.app-name=sha-$APP" --quiet) &>/dev/null || true
-
-  images="$("$DOCKER_BIN" images --filter "label=com.dokku.app-name=sha-$APP" --quiet)"
-  for image in $images; do
-    if [[ "$("$DOCKER_BIN" inspect --format '{{(index .RepoTags 0)}}' "$image" 2>/dev/null)" != "$IMAGE" ]]; then
-      "$DOCKER_BIN" rmi "$image" &>/dev/null || true
-    fi
-  done
-}
+source "$PLUGIN_AVAILABLE_PATH/apt/internal-functions"
 
 hook-apt-pre-build-buildpack() {
   declare APP="$1" SOURCECODE_WORK_DIR="$2"
   local IMAGE="dokku/$APP" DIR=/app
-  local COMMAND CONTENT CONTENT_SHA INJECT_PACKAGES
+  local CID COMMAND DOCKER_COMMIT_LABEL_ARGS
 
   if [[ -n "$SOURCECODE_WORK_DIR" ]]; then
     pushd "$SOURCECODE_WORK_DIR" >/dev/null
   fi
 
-  local APT_FILES=('apt-env' 'apt-preferences' 'apt-sources-list' 'apt-repositories' 'apt-debconf' 'apt-packages' 'dpkg-packages')
-  for file in "${APT_FILES[@]}"; do
-    if [[ -f "$file" ]]; then
-      INJECT_PACKAGES=true
-      local file_contents=$(<$file)
-      CONTENT="${CONTENT}\n${file}\n${file_contents}"
-      if [[ "$file" == "dpkg-packages" ]]; then
-        CONTENT="${CONTENT}$(date +%s)"
-      fi
-    fi
-  done
+  if [[ ! -f "dpkg-packages" ]]; then
+    return
+  fi
 
   if [[ -n "$SOURCECODE_WORK_DIR" ]]; then
     popd >/dev/null
   fi
 
-  if [[ "$INJECT_PACKAGES" != "true" ]]; then
-    return
-  fi
-
-  COMMAND=$(
-    cat <<EOF
-# $APP $IMAGE
-sleep 2
-export DEBIAN_FRONTEND=noninteractive
-if [ -f $DIR/apt-env ]; then
-    echo "-----> Sourcing apt env"
-    source $DIR/apt-env
-fi
-if [ -d $DIR/apt-preferences ]; then
-    echo "-----> Injecting apt preferences"
-    mv -v $DIR/apt-preferences /etc/apt/preferences.d/90customizations
-fi
-if [ -f $DIR/apt-sources-list ]; then
-    echo "-----> Using customized sources.list"
-    mv -v $DIR/apt-sources-list /etc/apt/sources.list
-fi
-if [ -f $DIR/apt-repositories ]; then
-    echo "-----> Updating package list"
-    apt-get update >/dev/null
-    echo "-----> Installing required apt transport packages"
-    apt-get install -y software-properties-common apt-transport-https
-    echo "-----> Installing custom apt repositories"
-    cat "$DIR/apt-repositories" | while read repository; do
-        if [ -n "\$repository" ]; then
-            add-apt-repository -y "\$repository"
-        fi
-    done
-fi
-if [ -f $DIR/apt-debconf ]; then
-    cat "$DIR/apt-debconf" | while read conf; do
-        if [ -n "\$conf" ]; then
-            echo \$conf | debconf-set-selections
-        fi
-    done
-fi
-if [ -f $DIR/apt-packages ]; then
-    PACKAGES=\$(cat "$DIR/apt-packages" | tr "\\n" " ")
-    echo "-----> Updating package list"
-    apt-get update >/dev/null
-    echo "-----> Injecting packages: \$PACKAGES"
-    apt-get install -y \$PACKAGES
-fi
-if [ -d $DIR/dpkg-packages ]; then
-    for pkg in $DIR/dpkg-packages/*.deb; do
-        echo "-----> Injecting package: \$pkg"
-        dpkg -i \$pkg
-    done
-fi
-sleep 1 # wait so that docker run has not exited before docker attach
-EOF
-  )
+  dokku_log_info1 "Creating extended app image with custom system packages"
+  COMMAND="$(fn-apt-command "$APP" "$IMAGE" "$DIR")"
+  CID=$(docker run -d "$IMAGE" /bin/bash -e -c "$COMMAND")
 
-  CONTENT_SHA="$(echo -n "$CONTENT" | sha256sum | cut -d " " -f 1)"
-  if [[ "$("$DOCKER_BIN" images --quiet "dokku/$APP:$CONTENT_SHA" 2>/dev/null)" != "" ]]; then
-    dokku_log_info1 "Compatible extended app image found, skipping system package installation"
-    "$DOCKER_BIN" tag "dokku/$APP:$CONTENT_SHA" "$IMAGE"
-    fn-clean-extended-app-images "$APP" "dokku/$APP:$CONTENT_SHA"
-    return
+  "$DOCKER_BIN" attach "$CID"
+  if test "$("$DOCKER_BIN" wait "$CID")" -ne 0; then
+    dokku_log_warn "Failure installing system packages"
+    return 1
   fi
 
-  dokku_log_info1 "Extending app image with custom system packages"
-  CID=$("$DOCKER_BIN" run -d "$IMAGE" /bin/bash -e -c "$COMMAND")
-  local DOCKER_COMMIT_LABEL_ARGS=("--change" "LABEL org.label-schema.schema-version=1.0" "--change" "LABEL org.label-schema.vendor=dokku" "--change" "LABEL com.dokku.app-name=sha-$APP" "--change" "LABEL $DOKKU_CONTAINER_LABEL=")
-
-  "$DOCKER_BIN" attach "$CID"
-  test "$("$DOCKER_BIN" wait "$CID")" -eq 0
+  DOCKER_COMMIT_LABEL_ARGS=("--change" "LABEL org.label-schema.schema-version=1.0" "--change" "LABEL org.label-schema.vendor=dokku" "--change" "LABEL com.dokku.app-name=$APP" "--change" "LABEL $DOKKU_CONTAINER_LABEL=")
   "$DOCKER_BIN" commit "${DOCKER_COMMIT_LABEL_ARGS[@]}" "$CID" "$IMAGE" >/dev/null
   "$DOCKER_BIN" rm "$CID" &>/dev/null || true
-  "$DOCKER_BIN" tag "$IMAGE" "dokku/$APP:$CONTENT_SHA"
-  fn-clean-extended-app-images "$APP" "dokku/$APP:$CONTENT_SHA"
 }
 
 hook-apt-pre-build-buildpack "$@"