Merge pull request #107 from qld-gov-au/develop

Develop to master - add CKAN 2.11 support
qld-gov-au · Feb 3, 2025 · 294e498 · 294e498
2 parents 9b0023f + 8f70736
commit 294e498
Show file tree

Hide file tree

Showing 26 changed files with 338 additions and 320 deletions.
diff --git a/.ahoy.yml b/.ahoy.yml
@@ -38,9 +38,17 @@ commands:
       ahoy title "Building and starting Docker containers"
       sh bin/docker-compose.sh up -d "$@"
       echo "Initialising database schema"
-      ahoy cli '$APP_DIR/bin/init.sh'
+      ahoy cli '"${APP_DIR}"/bin/init.sh'
       echo "Waiting for containers to start listening..."
-      ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
+      for i in `seq 1 60`; do
+        if (ahoy cli "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/ckan/5000'"); then
+          echo "CKAN became ready on attempt $i"
+          break
+        else
+          echo "CKAN not yet ready, retrying (attempt $i)..."
+          sleep 1
+        fi
+      done
       if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
       if sh bin/docker-compose.sh logs | grep -q "Exception"; then exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
@@ -66,7 +74,15 @@ commands:
 
   logs:
     usage: Show Docker logs.
-    cmd: sh bin/docker-compose.sh logs "$@"
+    cmd: |
+        ahoy title "Output logs"
+        # Loop through each container and wrap with github log groups
+        services=$(bin/docker-compose.sh ps --services)
+        for service in $services; do
+          echo "::group::$service"
+          sh bin/docker-compose.sh logs "$service"
+          echo "::endgroup::"
+        done
 
   pull:
     usage: Pull latest docker images.
@@ -77,9 +93,9 @@ commands:
     cmd: |
       CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan)
       if [ "${#}" -ne 0 \]; then
-        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"
+        docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate; cd $APP_DIR;'" $*"
       else
-        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'
+        docker exec $CKAN_CONTAINER sh -c '. "${APP_DIR}"/bin/activate && cd $APP_DIR && sh'
       fi
 
   doctor:
@@ -90,7 +106,7 @@ commands:
     usage: Install test site data.
     cmd: |
       ahoy title "Installing a fresh site"
-      ahoy cli '$APP_DIR/bin/init.sh && $APP_DIR/bin/create-test-data.sh'
+      ahoy cli '"${APP_DIR}"/bin/init.sh && "${APP_DIR}"/bin/create-test-data.sh'
 
   clean:
     usage: Remove containers and all build files.
@@ -126,13 +142,13 @@ commands:
     cmd: |
       docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/
       docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/
-      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI; $APP_DIR/bin/process-config.sh'
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli "${APP_DIR}"/bin/*; cp -v .docker/test.ini $CKAN_INI; "${APP_DIR}"/bin/process-config.sh'
 
   test-unit:
     usage: Run unit tests.
     cmd: |
       ahoy title 'Run unit tests'
-      ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \
+      ahoy cli 'pytest --ckan-ini=${CKAN_INI} --cov=ckanext "${APP_DIR}"/ckanext --junit-xml=test/junit/results.xml' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
@@ -142,13 +158,14 @@ commands:
       ahoy cli "rm -f test/screenshots/*"
       ahoy start-mailmock &
       sleep 5
+      JUNIT_OUTPUT="--junit --junit-directory=test/junit/"
       if [ "$BEHAVE_TAG" = "" ]; then
-        (ahoy cli "behave -k ${*:-test/features} --tags=smoke" \
-         && ahoy cli "behave -k ${*:-test/features}" \
+        (ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} --tags=smoke" \
+         && ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} --tags=-smoke"
         ) || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       else
         # run tests with the specified tag
-        ahoy cli "behave -k ${*:-test/features} --tags=$BEHAVE_TAG" \
+        ahoy cli "behave $JUNIT_OUTPUT -k ${*:-test/features} --tags=$BEHAVE_TAG" \
           || [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       fi
       ahoy stop-mailmock
@@ -157,7 +174,7 @@ commands:
     usage: Starts email mock server used for email BDD tests
     cmd: |
       ahoy title 'Starting mailmock'
-      ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout
+      ahoy cli 'mailmock -p 8025 -o "${APP_DIR}"/test/emails' # for debugging mailmock email output remove --no-stdout
 
   stop-mailmock:
     usage: Stops email mock server used for email BDD tests

diff --git a/.docker/Dockerfile-template.ckan b/.docker/Dockerfile-template.ckan
@@ -1,20 +1,34 @@
-FROM openknowledge/ckan-dev:{CKAN_VERSION}
+FROM ckan/ckan-dev:{CKAN_VERSION}
+
+# swap between root and unprivileged user
+ARG ORIGINAL_USER
+RUN ORIGINAL_USER=$(id -un)
 
 ARG SITE_URL=http://ckan:5000/
 ENV PYTHON_VERSION={PYTHON_VERSION}
 ENV CKAN_VERSION={CKAN_VERSION}
+ENV SOLR_VERSION={SOLR_VERSION}
 ENV CKAN_SITE_URL="${SITE_URL}"
 ENV PYTHON={PYTHON}
 
 WORKDIR "${APP_DIR}"
 
-ENV DOCKERIZE_VERSION v0.6.1
-RUN apk add --no-cache build-base \
-    && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
-        | tar -C /usr/local/bin -xzvf -
+COPY .docker/test.ini $CKAN_INI
+
+COPY . "${APP_DIR}"/
+
+USER root
+
+COPY bin/ckan_cli /usr/bin/
+
+RUN chmod +x "${APP_DIR}"/bin/*.sh /usr/bin/ckan_cli
 
-# Update urllib3 to fix urlopen bug
-RUN pip install -U urllib3
+RUN which ps || apt-get install -y procps
+
+# Install setuptools conditionally
+RUN if [ "$CKAN_VERSION" = "2.9" ]; then \
+        pip install "setuptools>=44.1.0,<71"; \
+    fi
 
 # Install CKAN.
 
@@ -25,15 +39,9 @@ RUN cd $SRC_DIR/ckan \
     && git reset --hard && git clean -f \
     && git checkout '{CKAN_GIT_VERSION}'
 
-COPY .docker/test.ini $CKAN_INI
-
-COPY . ${APP_DIR}/
-
-COPY bin/ckan_cli /usr/bin/
-
-RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli
+USER "$ORIGINAL_USER"
 
 # Init current extension.
-RUN ${APP_DIR}/bin/init-ext.sh
+RUN "${APP_DIR}"/bin/init-ext.sh
 
 CMD ["/srv/app/bin/serve.sh"]
diff --git a/.docker/test.ini b/.docker/test.ini
@@ -1,16 +1,3 @@
-#
-# CKAN - Pylons configuration
-#
-# These are some of the configuration options available for your CKAN
-# instance. Check the documentation in 'doc/configuration.rst' or at the
-# following URL for a description of what they do and the full list of
-# available options:
-#
-# http://docs.ckan.org/en/latest/maintaining/configuration.html
-#
-# The %(here)s variable will be replaced with the parent directory of this file
-#
-
 [DEFAULT]
 debug = false
 smtp_server = localhost:8025
@@ -30,13 +17,9 @@ full_stack = true
 cache_dir = /tmp/%(ckan.site_id)s/
 beaker.session.key = ckan
 
-# This is the secret token that the beaker library uses to hash the cookie sent
-# to the client. `paster make-config` generates a unique value for this each
-# time it generates a config file.
+SECRET_KEY = bSmgPpaxg2M+ZRes3u1TXwIcE
 beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE
 
-# `paster make-config` generates a unique value for this each time it generates
-# a config file.
 app_instance_uuid = 6e3daf8e-1c6b-443b-911f-c7ab4c5f9605
 
 # repoze.who config
@@ -67,6 +50,8 @@ ckan.auth.create_user_via_api = false
 ckan.auth.create_user_via_web = true
 ckan.auth.roles_that_cascade_to_sub_groups = admin
 ckan.auth.public_user_details = False
+ckan.auth.reveal_private_datasets = True
+ckan.auth.reveal_deleted_datasets = False
 
 
 ## Search Settings

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -26,52 +26,79 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ckan-version: ["2.10", 2.9]
+        ckan-version: ['2.10']
+        solr-version: ['8']
+        experimental: [false]
+        include:
+          - ckan-version: '2.11'
+            solr-version: '9'
+            experimental: false
+          - ckan-version: 'master'
+            solr-version: '9'
+            experimental: true  #master is unstable, good to know if we are compatible or not
 
-    name: Test on CKAN ${{ matrix.ckan-version }}
+    name: ${{ matrix.experimental && '**Fail_Ignored** ' || '' }} CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container: drevops/ci-runner:23.12.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
+      SOLR_VERSION: ${{ matrix.solr-version }}
 
     steps:
       # Patch https://github.com/actions/runner/issues/863
       - name: Preserve $HOME set in the container
         run: echo HOME=/root >> "$GITHUB_ENV"
 
       - uses: actions/checkout@v4
+        continue-on-error: ${{ matrix.experimental }}
         timeout-minutes: 2
 
       - name: Build
         run: bin/build.sh
         timeout-minutes: 15
+        continue-on-error: ${{ matrix.experimental }}
 
-      - name: Test lint
-        run: bin/test-lint.sh
-        timeout-minutes: 30
+      - name: Lint
+        run: ahoy lint
+        timeout-minutes: 10
+        continue-on-error: ${{ matrix.experimental }}
 
-      - name: Test
-        run: bin/test.sh
-        timeout-minutes: 30
+      - name: Unit test
+        run: ahoy test-unit
+        timeout-minutes: 10
+        continue-on-error: ${{ matrix.experimental }}
 
-      - name: Test BDD
-        run: bin/test-bdd.sh
+      - name: Scenario test
+        run: |
+          ahoy install-site
+          ahoy test-bdd
         timeout-minutes: 40
+        continue-on-error: ${{ matrix.experimental }}
 
       - name: Retrieve logs
         if: always()
-        run: bin/get-logs.sh
+        run: ahoy logs
         timeout-minutes: 1
+        continue-on-error: ${{ matrix.experimental }}
 
-      - name: Retrieve screenshots
-        if: failure()
+      - name: Retrieve results
+        if: always()
         run: bin/process-artifacts.sh
         timeout-minutes: 1
+        continue-on-error: ${{ matrix.experimental }}
+
+      - name: Test Summary
+        uses: test-summary/action@v2
+        continue-on-error: ${{ matrix.experimental }}
+        with:
+          paths: "/tmp/artifacts/junit/*.xml"
+        if: always()
 
       - name: Upload screenshots
-        if: failure()
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: CKAN ${{ matrix.ckan-version }} screenshots
           path: /tmp/artifacts/behave/screenshots
         timeout-minutes: 1
+        continue-on-error: ${{ matrix.experimental }}
diff --git a/bin/build.sh b/bin/build.sh
@@ -14,23 +14,18 @@ sed -i -e "s/##//" docker-compose.yml
 # Pull the latest images.
 ahoy pull
 
-PYTHON=python
+PYTHON=python3
+PYTHON_VERSION=py3
 
 CKAN_GIT_VERSION=$CKAN_VERSION
 CKAN_GIT_ORG=qld-gov-au
 
-if [ "$CKAN_VERSION" = "2.10" ]; then
-    CKAN_GIT_VERSION=ckan-2.10.1-qgov.10
-    PYTHON_VERSION=py3
-    PYTHON="${PYTHON}3"
-else
-    CKAN_GIT_VERSION=ckan-2.9.9-qgov.3
-    if [ "$CKAN_VERSION" = "2.9-py2" ]; then
-        PYTHON_VERSION=py2
-    else
-        PYTHON_VERSION=py3
-        PYTHON="${PYTHON}3"
-    fi
+if [ "$CKAN_VERSION" = "2.11" ]; then
+    CKAN_GIT_VERSION=ckan-2.11.1-qgov.2
+elif [ "$CKAN_VERSION" = "2.10" ]; then
+    CKAN_GIT_VERSION=ckan-2.10.5-qgov.5
+elif [ "$CKAN_VERSION" = "master" ]; then
+    CKAN_GIT_ORG=ckan
 fi
 
 sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \