Merge pull request DefectDojo#10678 from DefectDojo/release/2.37.0

Release: Merge release into master from: release/2.37.0
kiblik · Aug 5, 2024 · 86e2961 · 86e2961
2 parents 73dddf6 + 3f20b29
commit 86e2961
Show file tree

Hide file tree

Showing 761 changed files with 24,039 additions and 19,139 deletions.
diff --git a/.github/renovate.json b/.github/renovate.json
@@ -7,10 +7,7 @@
   "baseBranches": ["dev"],
   "rebaseWhen": "conflicted",
   "ignorePaths": ["requirements.txt", "requirements-lint.txt", "components/package.json", "components/package-lock.json", "dojo/components/yarn.lock", "dojo/components/package.json", "Dockerfile**"],
-  "ignoreDeps": [
-    "mysql",
-    "rabbitmq"
-  ],
+  "ignoreDeps": [],
   "packageRules": [{
     "packagePatterns": ["*"],
     "commitMessageExtra": "from {{currentVersion}} to {{#if isMajor}}v{{{newMajor}}}{{else}}{{#if isSingleVersion}}v{{{toVersion}}}{{else}}{{{newValue}}}{{/if}}{{/if}}",

diff --git a/.github/workflows/fetch-oas.yml b/.github/workflows/fetch-oas.yml
@@ -33,7 +33,7 @@ jobs:
              docker images
 
       - name: Start Dojo
-        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env up --no-deps -d postgres nginx uwsgi
+        run: docker compose up --no-deps -d postgres nginx uwsgi
         env:
           DJANGO_VERSION: ${{ env.release_version }}-alpine
           NGINX_VERSION: ${{ env.release_version }}-alpine
@@ -44,11 +44,11 @@ jobs:
 
       - name: Logs
         if: always()
-        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env logs --tail="2500"
+        run: docker compose logs --tail="2500"
 
       - name: Shutdown
         if: always()
-        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env down
+        run: docker compose down
 
       - name: Upload oas.${{ matrix.file-type }} as artifact
         uses:  actions/upload-artifact@v3

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -36,7 +36,6 @@ jobs:
           "tests/tool_config.py",
           "openapi-validatator",
         ]
-        profile: ["postgres-rabbitmq", "postgres-redis"]
         os: [alpine, debian]
       fail-fast: false
 
@@ -59,39 +58,30 @@ jobs:
       - name: Set integration-test mode
         run: ln -s docker-compose.override.integration_tests.yml docker-compose.override.yml
 
-      # phased startup with PostgreSQL and RabbitMQ so we can use the exit code from integrationtest container
-      - name: Start Dojo PostgreSQL + RabbitMQ
-        if: matrix.profile == 'postgres-rabbitmq'
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps -d postgres nginx celerybeat celeryworker mailhog uwsgi rabbitmq
-        env:
-          DJANGO_VERSION: ${{ matrix.os }}
-          NGINX_VERSION: ${{ matrix.os }}
-
-      - name: Start Dojo PostgreSQL + Redis
-        if: matrix.profile == 'postgres-redis'
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps -d postgres nginx celerybeat celeryworker mailhog uwsgi redis
+      - name: Start Dojo 
+        run: docker compose up --no-deps -d postgres nginx celerybeat celeryworker mailhog uwsgi redis
         env:
           DJANGO_VERSION: ${{ matrix.os }}
           NGINX_VERSION: ${{ matrix.os }}
 
       - name: Initialize
         timeout-minutes: 10
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps --exit-code-from initializer initializer
+        run: docker compose up --no-deps --exit-code-from initializer initializer
         env:
           DJANGO_VERSION: ${{ matrix.os }}
           NGINX_VERSION: ${{ matrix.os }}
 
       - name: Integration tests
         timeout-minutes: 10
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env up --no-deps --exit-code-from integration-tests integration-tests
+        run: docker compose up --no-deps --exit-code-from integration-tests integration-tests
         env:
           DD_INTEGRATION_TEST_FILENAME: ${{ matrix.test-case }}
           INTEGRATION_TESTS_VERSION: debian
 
       - name: Logs
         if: always()
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env logs --tail="2500"
+        run: docker compose logs --tail="2500"
 
       - name: Shutdown
         if: always()
-        run: docker compose --profile ${{ matrix.profile }} --env-file ./docker/environments/${{ matrix.profile }}.env down
+        run: docker compose down
diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml
@@ -5,28 +5,14 @@ on:
 
 env:
   DD_HOSTNAME: defectdojo.default.minikube.local
-  HELM_RABBIT_BROKER_SETTINGS: " \
-    --set redis.enabled=false \
-    --set rabbitmq.enabled=true \
-    --set celery.broker=rabbitmq \
-    --set createRabbitMqSecret=true \
-    "
   HELM_REDIS_BROKER_SETTINGS: " \
     --set redis.enabled=true \
-    --set rabbitmq.enabled=false \
     --set celery.broker=redis \
     --set createRedisSecret=true \
     "
-  HELM_MYSQL_DATABASE_SETTINGS: " \
-    --set database=mysql \
-    --set postgresql.enabled=false \
-    --set mysql.enabled=true \
-    --set createMysqlSecret=true \
-    "
   HELM_PG_DATABASE_SETTINGS: " \
     --set database=postgresql \
     --set postgresql.enabled=true \
-    --set mysql.enabled=false \
     --set createPostgresqlSecret=true \
    "
 jobs:
@@ -44,23 +30,6 @@ jobs:
             brokers: redis
             k8s: 'v1.26.11'
             os: debian
-          - databases: mysql
-            brokers: rabbit
-            k8s: 'v1.26.11'
-            os: debian
-          - databases: pgsql
-            brokers: rabbit
-            k8s: 'v1.29.2'
-            os: debian
-          - databases: mysql
-            brokers: redis
-            k8s: 'v1.29.2'
-            os: debian
-          - databases: pgsql
-            brokers: rabbit
-            k8s: 'v1.29.2'
-            os: alpine
-
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -99,9 +68,7 @@ jobs:
         id: set
         run: |-
               echo "pgsql=${{ env.HELM_PG_DATABASE_SETTINGS }}" >> $GITHUB_ENV
-              echo "mysql=${{ env.HELM_MYSQL_DATABASE_SETTINGS }}" >> $GITHUB_ENV
               echo "redis=${{ env.HELM_REDIS_BROKER_SETTINGS }}" >> $GITHUB_ENV
-              echo "rabbit=${{ env.HELM_RABBIT_BROKER_SETTINGS }}" >> $GITHUB_ENV
 
       - name: Deploying Djano application with ${{ matrix.databases }} ${{ matrix.brokers }}
         timeout-minutes: 10 

diff --git a/.github/workflows/release-1-create-pr.yml b/.github/workflows/release-1-create-pr.yml
@@ -51,9 +51,9 @@ jobs:
 
       - name: Update version numbers in key files
         run: |
-          sed -ri "s/__version__ = '.*'/__version__ = '${{ github.event.inputs.release_number }}'/" dojo/__init__.py
-          sed -ri "s/\"version\": \".*\"/\"version\": \"${{ github.event.inputs.release_number }}\"/" components/package.json
-          sed -ri "s/appVersion: \".*\"/appVersion: \"${{ github.event.inputs.release_number }}\"/" helm/defectdojo/Chart.yaml
+          sed -ri 's/__version__ = ".*"/__version__ = "${{ github.event.inputs.release_number }}"/' dojo/__init__.py
+          sed -ri 's/"version": ".*"/"version": "${{ github.event.inputs.release_number }}"/' components/package.json
+          sed -ri 's/appVersion: ".*"/appVersion: "${{ github.event.inputs.release_number }}"/' helm/defectdojo/Chart.yaml
 
           if grep "\-dev" helm/defectdojo/Chart.yaml; then
               echo "x.y.z-dev found in Chart.yaml, probably releasing a new minor version"

diff --git a/.github/workflows/release-3-master-into-dev.yml b/.github/workflows/release-3-master-into-dev.yml
@@ -44,9 +44,9 @@ jobs:
 
       - name: Update version numbers in key files
         run: |
-          sed -ri "s/__version__ = '.*'/__version__ = '${{ github.event.inputs.release_number_dev }}'/" dojo/__init__.py
-          sed -ri "s/appVersion: \".*\"/appVersion: \"${{ github.event.inputs.release_number_dev }}\"/" helm/defectdojo/Chart.yaml
-          sed -ri "s/\"version\": \".*\"/\"version\": \"${{ github.event.inputs.release_number_dev }}\"/" components/package.json
+          sed -ri 's/__version__ = ".*"/__version__ = "${{ github.event.inputs.release_number_dev }}"/' dojo/__init__.py
+          sed -ri 's/"version": ".*"/"version": "${{ github.event.inputs.release_number_dev }}"/' components/package.json
+          sed -ri 's/appVersion: ".*"/appVersion: "${{ github.event.inputs.release_number_dev }}"/' helm/defectdojo/Chart.yaml
           CURRENT_CHART_VERSION=$(grep -oP 'version: (\K\S*)?' helm/defectdojo/Chart.yaml | head -1)
           sed -ri "0,/version/s/version: \S+/$(echo "version: $CURRENT_CHART_VERSION" | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}')-dev/" helm/defectdojo/Chart.yaml
       

diff --git a/.github/workflows/rest-framework-tests.yml b/.github/workflows/rest-framework-tests.yml
@@ -35,19 +35,19 @@ jobs:
 
       # phased startup so we can use the exit code from unit test container
       - name: Start Postgres
-        run: docker compose --env-file ./docker/environments/postgres-redis.env up -d postgres
+        run: docker compose up -d postgres
 
       # no celery or initializer needed for unit tests
       - name: Unit tests
         timeout-minutes: 10
-        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env up --no-deps --exit-code-from uwsgi uwsgi
+        run: docker compose up --no-deps --exit-code-from uwsgi uwsgi
         env:
           DJANGO_VERSION: ${{ matrix.os }}
 
       - name: Logs
         if: failure()
-        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env logs --tail="2500" uwsgi
+        run: docker compose logs --tail="2500" uwsgi
 
       - name: Shutdown
         if: always()
-        run: docker compose --profile postgres-redis --env-file ./docker/environments/postgres-redis.env down
+        run: docker compose down
diff --git a/Dockerfile.django-alpine b/Dockerfile.django-alpine
@@ -5,24 +5,24 @@
 # Dockerfile.nginx to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.9-alpine3.20@sha256:df44c0c0761ddbd6388f4549cab42d24d64d257c2a960ad5b276bb7dab9639c7 as base
-FROM base as build
+FROM python:3.11.9-alpine3.20@sha256:df44c0c0761ddbd6388f4549cab42d24d64d257c2a960ad5b276bb7dab9639c7 AS base
+FROM base AS build
 WORKDIR /app
 RUN \
   apk update && \
   apk add --no-cache \
     gcc \
     build-base \
     bind-tools \
-    mysql-client \
-    mariadb-dev \
-    postgresql14-client \
+    postgresql16-client \
     xmlsec \
     git \
     util-linux \
     curl-dev \
     openssl \
     libffi-dev \
+    python3-dev \
+    libpq-dev \
     && \
     rm -rf /var/cache/apk/* && \
   true
@@ -31,25 +31,23 @@ COPY requirements.txt ./
 # https://github.com/unbit/uwsgi/issues/1318#issuecomment-542238096
 RUN CPUCOUNT=1 pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 
-FROM base as django-alpine
+FROM base AS django-alpine
 WORKDIR /app
 ARG uid=1001
 ARG gid=1337
 ARG appuser=defectdojo
-ENV appuser ${appuser}
+ENV appuser=${appuser}
 RUN \
   apk update && \
   apk add --no-cache \
     openjpeg \
     jpeg \
     tiff \
     bind-tools \
-    mysql-client \
-    mariadb-dev \
     xmlsec \
     git \
     util-linux \
-    postgresql14-client \
+    postgresql16-client \
     curl-dev \
     openssl \
     # needed for integration-tests
@@ -136,5 +134,5 @@ ENV \
   DD_UWSGI_NUM_OF_THREADS="2"
 ENTRYPOINT ["/entrypoint-uwsgi.sh"]
 
-FROM django-alpine as django-unittests
-COPY unittests/ ./unittests/
+FROM django-alpine AS django-unittests
+COPY unittests/ ./unittests/
diff --git a/Dockerfile.django-debian b/Dockerfile.django-debian
@@ -5,17 +5,15 @@
 # Dockerfile.nginx to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e as base
-FROM base as build
+FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e AS base
+FROM base AS build
 WORKDIR /app
 RUN \
   apt-get -y update && \
   apt-get -y install --no-install-recommends \
     gcc \
     build-essential \
     dnsutils \
-    default-mysql-client \
-    libmariadb-dev-compat \
     libpq-dev \
     postgresql-client \
     xmlsec1 \
@@ -32,12 +30,12 @@ COPY requirements.txt ./
 # https://github.com/unbit/uwsgi/issues/1318#issuecomment-542238096
 RUN CPUCOUNT=1 pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 
-FROM base as django
+FROM base AS django
 WORKDIR /app
 ARG uid=1001
 ARG gid=1337
 ARG appuser=defectdojo
-ENV appuser ${appuser}
+ENV appuser=${appuser}
 RUN \
   apt-get -y update && \
   # ugly fix to install postgresql-client without errors
@@ -48,8 +46,6 @@ RUN \
     libjpeg62 \
     libtiff6 \
     dnsutils \
-    default-mysql-client \
-    libmariadb3 \
     xmlsec1 \
     git \
     uuid-runtime \
@@ -141,5 +137,5 @@ ENV \
   DD_UWSGI_NUM_OF_THREADS="2"
 ENTRYPOINT ["/entrypoint-uwsgi.sh"]
 
-FROM django as django-unittests
+FROM django AS django-unittests
 COPY unittests/ ./unittests/
diff --git a/Dockerfile.integration-tests-debian b/Dockerfile.integration-tests-debian
@@ -1,8 +1,8 @@
 
 # code: language=Dockerfile
 
-FROM openapitools/openapi-generator-cli:v7.6.0@sha256:f86ca824293602b71b9b66683cc0011f8ff963858bd853621c554ff5cc7dd1d5 as openapitools
-FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e as build
+FROM openapitools/openapi-generator-cli:v7.7.0@sha256:99924315933d49e7b33a7d2074bb2b64fc8def8f74519939036e24eb48f00336 AS openapitools
+FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e AS build
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -76,4 +76,4 @@ ENV \
   DD_ADMIN_USER=admin \
   DD_ADMIN_PASSWORD='' \
   DD_BASE_URL="http://localhost:8080/"
-CMD ["/entrypoint-integration-tests.sh"]
+CMD ["/entrypoint-integration-tests.sh"]
diff --git a/Dockerfile.nginx-alpine b/Dockerfile.nginx-alpine
@@ -5,24 +5,24 @@
 # Dockerfile.django-alpine to use the caching mechanism of Docker.
 
 # Ref: https://devguide.python.org/#branchstatus
-FROM python:3.11.9-alpine3.20@sha256:df44c0c0761ddbd6388f4549cab42d24d64d257c2a960ad5b276bb7dab9639c7 as base
-FROM base as build
+FROM python:3.11.9-alpine3.20@sha256:df44c0c0761ddbd6388f4549cab42d24d64d257c2a960ad5b276bb7dab9639c7 AS base
+FROM base AS build
 WORKDIR /app
 RUN \
   apk update && \
   apk add --no-cache \
     gcc \
     build-base \
     bind-tools \
-    mysql-client \
-    mariadb-dev \
-    postgresql14-client \
+    postgresql16-client \
     xmlsec \
     git \
     util-linux \
     curl-dev \
     openssl \
     libffi-dev \
+    python3-dev \
+    libpq-dev \
     && \
     rm -rf /var/cache/apk/* && \
   true
@@ -34,7 +34,7 @@ RUN CPUCOUNT=1 pip3 wheel --wheel-dir=/tmp/wheels -r ./requirements.txt
 FROM build AS collectstatic
 
 # Node installation from https://github.com/nodejs/docker-node
-ENV NODE_VERSION 20.11.0
+ENV NODE_VERSION=20.11.0
 
 RUN addgroup -g 1000 node \
     && adduser -u 1000 -G node -s /bin/sh -D node \
@@ -103,7 +103,7 @@ RUN addgroup -g 1000 node \
   && node --version \
   && npm --version
 
-ENV YARN_VERSION 1.22.19
+ENV YARN_VERSION=1.22.19
 
 RUN apk add --no-cache --virtual .build-deps-yarn curl gnupg tar \
   && for key in \
@@ -140,7 +140,7 @@ COPY manage.py ./
 COPY dojo/ ./dojo/
 RUN env DD_SECRET_KEY='.' python3 manage.py collectstatic --noinput && true
 
-FROM nginx:1.27.0-alpine@sha256:a45ee5d042aaa9e81e013f97ae40c3dda26fbe98f22b6251acdf28e579560d55
+FROM nginx:1.27.0-alpine@sha256:208b70eefac13ee9be00e486f79c695b15cef861c680527171a27d253d834be9
 ARG uid=1001
 ARG appuser=defectdojo
 COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/