[minor] add Dockerfile healthchecks to some core services (#370)

* Create HEALTHCHECK entries for core services

* Ensure secret is set for cantaloupe API access

* Add test to make sure healthcheck works

Allow fpm status from ipv6 localhost

* Add more tests

* update README

* Update README.md

* Healthchecks review (#375)

* Replace legacy cont-init.d for Cantaloupe service

Changes the cantaloupe service init such that it respects service
dependencies.

This insures that the precedence for container environment follows what
is outlined in the container-environment service.

* Test compose projects require names.

* Bump github action upload artifact

* Add names to docker compose tests

---------

Co-authored-by: Nigel Banks <nigelgbanks@users.noreply.github.com>

.github/workflows/grype.yml

@@ -33,8 +33,8 @@ jobs:
         with:
           cache-read-only: false
           arguments: ${{ inputs.image }}:grype ${{ inputs.digest }} --info
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: Grype Reports
+          name: ${{ inputs.image }} Grype Reports
           path: build/**/*-grype.*

activemq/Dockerfile

@@ -36,3 +36,9 @@ COPY --link rootfs /
 
 RUN create-service-user.sh --name activemq && \
     cleanup.sh
+
+HEALTHCHECK CMD curl -s \
+    -u admin:"$(cat /var/run/s6/container_environment/ACTIVEMQ_WEB_ADMIN_PASSWORD)" \
+    -H origin:localhost \
+    "http://localhost:8161/api/jolokia/read/org.apache.activemq:type=Broker,brokerName=localhost,service=Health/CurrentStatus" \
+    | jq .value | grep -q Good

activemq/tests/ServiceHealthcheck/build.gradle.kts

@@ -0,0 +1,54 @@
+import plugins.TestsPlugin.DockerComposeUp
+import plugins.TestsPlugin.DockerComposeUp.Companion.pool
+import java.lang.Thread.sleep
+import java.time.Duration.ofSeconds
+import java.util.concurrent.CompletableFuture.supplyAsync
+import java.io.ByteArrayOutputStream
+
+tasks.named<DockerComposeUp>("test") {
+    doFirst {
+        supplyAsync(
+            {
+                val maxAttempts = 10
+                val delayBetweenAttempts = 5000L // 5 seconds in milliseconds
+                var attempt = 0
+                var foundHealthyService = false
+
+                while (attempt < maxAttempts) {
+                    attempt++
+                    val outputStream = ByteArrayOutputStream()
+                    project.exec {
+                        commandLine = baseArguments + listOf("ps", "--all")
+                        standardOutput = outputStream
+                        workingDir = project.projectDir
+                    }
+                    val output = outputStream.toString()
+
+                    val healthyServicePattern = """(?m)^.+\s+.+\s+Up \d+ seconds \(healthy\).*$""".toRegex()
+                    foundHealthyService = output.lines().any { line ->
+                        healthyServicePattern.matches(line)
+                    }
+
+                    if (foundHealthyService) {
+                        project.exec {
+                            commandLine = baseArguments + listOf("stop")
+                            standardOutput = outputStream
+                            workingDir = project.projectDir
+                        }
+                        break
+                    }
+
+                    if (attempt < maxAttempts) {
+                        println("No healthy service found. Retrying in ${delayBetweenAttempts / 1000} seconds...")
+                        sleep(delayBetweenAttempts)
+                    }
+                }
+
+                // Throw an exception if no healthy service was found after all attempts
+                if (!foundHealthyService) {
+                    throw GradleException("No service is marked as healthy in docker compose ps output after $maxAttempts attempts.")
+                }
+            }, pool
+        )
+    }
+}

activemq/tests/ServiceHealthcheck/docker-compose.yml

@@ -0,0 +1,15 @@
+---
+
+# Common to all services
+x-common: &common
+  restart: "no"
+
+name: activemq-servicehealthcheck
+services:
+  activemq:
+    <<: *common
+    image: ${ACTIVEMQ:-islandora/activemq:local}
+    volumes:
+      - ./test.sh:/test.sh # Test to run.
+    command:
+      - /test.sh # Run test and exit.

activemq/tests/ServiceHealthcheck/test.sh

@@ -0,0 +1,13 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+on_terminate() {
+    echo "Termination signal received. Exiting..."
+    exit 0
+}
+trap 'on_terminate' SIGTERM
+
+sleep 60
+
+# The kotlin check should be stopping this container
+exit 1

cantaloupe/Dockerfile

@@ -83,9 +83,9 @@ ENV \
     CANTALOUPE_ENDPOINT_ADMIN_ENABLED="false" \
     CANTALOUPE_ENDPOINT_ADMIN_SECRET= \
     CANTALOUPE_ENDPOINT_ADMIN_USERNAME="admin" \
-    CANTALOUPE_ENDPOINT_API_ENABLED="false" \
+    CANTALOUPE_ENDPOINT_API_ENABLED="true" \
     CANTALOUPE_ENDPOINT_API_SECRET= \
-    CANTALOUPE_ENDPOINT_API_USERNAME= \
+    CANTALOUPE_ENDPOINT_API_USERNAME=islandora \
     CANTALOUPE_ENDPOINT_HEALTH_DEPENDENCY_CHECK="false" \
     CANTALOUPE_ENDPOINT_IIIF_1_ENABLED="false" \
     CANTALOUPE_ENDPOINT_IIIF_2_ENABLED="true" \
@@ -271,3 +271,7 @@ ENV \
     CANTALOUPE_TEMP_PATHNAME=
 
 COPY --link rootfs /
+
+HEALTHCHECK CMD curl -s \
+  -u "${CANTALOUPE_ENDPOINT_API_USERNAME}:$(cat /var/run/s6/container_environment/CANTALOUPE_ENDPOINT_API_SECRET)" \
+  http://localhost:8182/health | jq .color | grep -q GREEN

cantaloupe/README.md

@@ -67,9 +67,9 @@ additional settings, volumes, ports, etc.
 | CANTALOUPE_ENDPOINT_ADMIN_ENABLED                                                      | "false"                                                          |
 | CANTALOUPE_ENDPOINT_ADMIN_SECRET                                                       |                                                                  |
 | CANTALOUPE_ENDPOINT_ADMIN_USERNAME                                                     | "admin"                                                          |
-| CANTALOUPE_ENDPOINT_API_ENABLED                                                        | "false"                                                          |
-| CANTALOUPE_ENDPOINT_API_SECRET                                                         |                                                                  |
-| CANTALOUPE_ENDPOINT_API_USERNAME                                                       |                                                                  |
+| CANTALOUPE_ENDPOINT_API_ENABLED                                                        | "true"                                                           |
+| CANTALOUPE_ENDPOINT_API_SECRET                                                         | random 16 char string                                            |
+| CANTALOUPE_ENDPOINT_API_USERNAME                                                       | "islandora"                                                      |
 | CANTALOUPE_ENDPOINT_HEALTH_DEPENDENCY_CHECK                                            | "false"                                                          |
 | CANTALOUPE_ENDPOINT_IIIF_1_ENABLED                                                     | "false"                                                          |
 | CANTALOUPE_ENDPOINT_IIIF_2_ENABLED                                                     | "true"                                                           |

cantaloupe/rootfs/etc/s6-overlay/s6-rc.d/cantaloupe-defaults/type

@@ -0,0 +1 @@
+oneshot

cantaloupe/rootfs/etc/s6-overlay/s6-rc.d/cantaloupe-defaults/up

@@ -0,0 +1 @@
+/etc/s6-overlay/scripts/cantaloupe-defaults.sh

cantaloupe/rootfs/etc/s6-overlay/scripts/cantaloupe-defaults.sh

@@ -0,0 +1,9 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+set -ex
+
+# Set the default value for CANTALOUPE_ENDPOINT_API_SECRET if none provided.
+DEFAULT_SECRET=$(openssl rand -hex 16)
+cat <<EOF | /usr/local/bin/confd-import-environment.sh
+CANTALOUPE_ENDPOINT_API_SECRET="{{ getenv "CANTALOUPE_ENDPOINT_API_SECRET" "${DEFAULT_SECRET}" }}"
+EOF

cantaloupe/tests/ServiceHealthcheck/build.gradle.kts

@@ -0,0 +1,54 @@
+import plugins.TestsPlugin.DockerComposeUp
+import plugins.TestsPlugin.DockerComposeUp.Companion.pool
+import java.lang.Thread.sleep
+import java.time.Duration.ofSeconds
+import java.util.concurrent.CompletableFuture.supplyAsync
+import java.io.ByteArrayOutputStream
+
+tasks.named<DockerComposeUp>("test") {
+    doFirst {
+        supplyAsync(
+            {
+                val maxAttempts = 10
+                val delayBetweenAttempts = 5000L // 5 seconds in milliseconds
+                var attempt = 0
+                var foundHealthyService = false
+
+                while (attempt < maxAttempts) {
+                    attempt++
+                    val outputStream = ByteArrayOutputStream()
+                    project.exec {
+                        commandLine = baseArguments + listOf("ps", "--all")
+                        standardOutput = outputStream
+                        workingDir = project.projectDir
+                    }
+                    val output = outputStream.toString()
+
+                    val healthyServicePattern = """(?m)^.+\s+.+\s+Up \d+ seconds \(healthy\).*$""".toRegex()
+                    foundHealthyService = output.lines().any { line ->
+                        healthyServicePattern.matches(line)
+                    }
+
+                    if (foundHealthyService) {
+                        project.exec {
+                            commandLine = baseArguments + listOf("stop")
+                            standardOutput = outputStream
+                            workingDir = project.projectDir
+                        }
+                        break
+                    }
+
+                    if (attempt < maxAttempts) {
+                        println("No healthy service found. Retrying in ${delayBetweenAttempts / 1000} seconds...")
+                        sleep(delayBetweenAttempts)
+                    }
+                }
+
+                // Throw an exception if no healthy service was found after all attempts
+                if (!foundHealthyService) {
+                    throw GradleException("No service is marked as healthy in docker compose ps output after $maxAttempts attempts.")
+                }
+            }, pool
+        )
+    }
+}

cantaloupe/tests/ServiceHealthcheck/docker-compose.yml

@@ -0,0 +1,15 @@
+---
+
+# Common to all services
+x-common: &common
+  restart: "no"
+
+name: cantaloupe-servicehealthcheck
+services:
+  cantaloupe:
+    <<: *common
+    image: ${CANTALOUPE:-islandora/cantaloupe:local}
+    volumes:
+      - ./test.sh:/test.sh # Test to run.
+    command:
+      - /test.sh # Run test and exit.

cantaloupe/tests/ServiceHealthcheck/test.sh

@@ -0,0 +1,13 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+on_terminate() {
+    echo "Termination signal received. Exiting..."
+    exit 0
+}
+trap 'on_terminate' SIGTERM
+
+sleep 60
+
+# The kotlin check should be stopping this container
+exit 1

ci/healthcheck.gradle.kts

@@ -0,0 +1,55 @@
+import plugins.TestsPlugin.DockerComposeUp
+import plugins.TestsPlugin.DockerComposeUp.Companion.pool
+import java.lang.Thread.sleep
+import java.time.Duration.ofSeconds
+import java.util.concurrent.CompletableFuture.supplyAsync
+import java.io.ByteArrayOutputStream
+
+tasks.named<DockerComposeUp>("test") {
+    doFirst {
+        supplyAsync(
+            {
+                val maxAttempts = 10
+                val delayBetweenAttempts = 5000L // 5 seconds in milliseconds
+                var attempt = 0
+                var foundHealthyService = false
+
+                while (attempt < maxAttempts) {
+                    attempt++
+                    val outputStream = ByteArrayOutputStream()
+                    project.exec {
+                        commandLine = baseArguments + listOf("ps", "--all")
+                        standardOutput = outputStream
+                        workingDir = project.projectDir
+                    }
+                    val output = outputStream.toString()
+
+                    val healthyServicePattern = """(?m)^.+\s+.+\s+Up \d+ seconds \(healthy\).*$""".toRegex()
+                    foundHealthyService = output.lines().any { line ->
+                        healthyServicePattern.matches(line)
+                    }
+
+                    if (foundHealthyService) {
+                        println("Service is healthy. Exiting test...")
+                        project.exec {
+                            commandLine = baseArguments + listOf("stop")
+                            standardOutput = outputStream
+                            workingDir = project.projectDir
+                        }
+                        break
+                    }
+
+                    if (attempt < maxAttempts) {
+                        println("No healthy service found. Retrying in ${delayBetweenAttempts / 1000} seconds...")
+                        sleep(delayBetweenAttempts)
+                    }
+                }
+
+                // Throw an exception if no healthy service was found after all attempts
+                if (!foundHealthyService) {
+                    throw GradleException("No service is marked as healthy in docker compose ps output after $maxAttempts attempts.")
+                }
+            }, pool
+        )
+    }
+}

fcrepo6/Dockerfile

@@ -70,3 +70,5 @@ ENV \
 COPY --link rootfs /
 
 RUN chown -R tomcat:tomcat /opt/tomcat
+
+HEALTHCHECK CMD curl -s http://localhost:8080/fcrepo/rest/fcr:systeminfo | jq -e .version

fcrepo6/tests/ServiceHealthcheck/build.gradle.kts

@@ -0,0 +1,54 @@
+import plugins.TestsPlugin.DockerComposeUp
+import plugins.TestsPlugin.DockerComposeUp.Companion.pool
+import java.lang.Thread.sleep
+import java.time.Duration.ofSeconds
+import java.util.concurrent.CompletableFuture.supplyAsync
+import java.io.ByteArrayOutputStream
+
+tasks.named<DockerComposeUp>("test") {
+    doFirst {
+        supplyAsync(
+            {
+                val maxAttempts = 10
+                val delayBetweenAttempts = 5000L // 5 seconds in milliseconds
+                var attempt = 0
+                var foundHealthyService = false
+
+                while (attempt < maxAttempts) {
+                    attempt++
+                    val outputStream = ByteArrayOutputStream()
+                    project.exec {
+                        commandLine = baseArguments + listOf("ps", "--all")
+                        standardOutput = outputStream
+                        workingDir = project.projectDir
+                    }
+                    val output = outputStream.toString()
+
+                    val healthyServicePattern = """(?m)^.+\s+.+\s+Up \d+ seconds \(healthy\).*$""".toRegex()
+                    foundHealthyService = output.lines().any { line ->
+                        healthyServicePattern.matches(line)
+                    }
+
+                    if (foundHealthyService) {
+                        project.exec {
+                            commandLine = baseArguments + listOf("stop")
+                            standardOutput = outputStream
+                            workingDir = project.projectDir
+                        }
+                        break
+                    }
+
+                    if (attempt < maxAttempts) {
+                        println("No healthy service found. Retrying in ${delayBetweenAttempts / 1000} seconds...")
+                        sleep(delayBetweenAttempts)
+                    }
+                }
+
+                // Throw an exception if no healthy service was found after all attempts
+                if (!foundHealthyService) {
+                    throw GradleException("No service is marked as healthy in docker compose ps output after $maxAttempts attempts.")
+                }
+            }, pool
+        )
+    }
+}

fcrepo6/tests/ServiceHealthcheck/docker-compose.yml

@@ -0,0 +1,19 @@
+---
+
+# Common to all services
+x-common: &common
+  restart: "no"
+
+name: fcrepo6-servicehealthcheck
+services:
+  activemq:
+    <<: *common
+    image: ${ACTIVEMQ:-islandora/activemq:local}
+  fcrepo6:
+    volumes:
+      - ./test.sh:/test.sh # Test to run.
+    command:
+      - /test.sh # Run test and exit.
+    image: ${FCREPO6:-islandora/fcrepo6:local}
+    depends_on:
+      - activemq

fcrepo6/tests/ServiceHealthcheck/test.sh

@@ -0,0 +1,13 @@
+#!/command/with-contenv bash
+# shellcheck shell=bash
+
+on_terminate() {
+    echo "Termination signal received. Exiting..."
+    exit 0
+}
+trap 'on_terminate' SIGTERM
+
+sleep 60
+
+# The kotlin check should be stopping this container
+exit 1

mariadb/Dockerfile

@@ -33,3 +33,5 @@ ENV \
     MYSQL_TRANSACTION_ISOLATION=READ-COMMITTED
 
 COPY --link rootfs /
+
+HEALTHCHECK CMD mysqladmin ping --socket=/var/run/mysqld/mysqld.sock | grep -q alive

mariadb/tests/ServiceHealthcheck/build.gradle.kts

@@ -0,0 +1 @@
+../../../ci/healthcheck.gradle.kts