Merge branch 'geopython:master' into staging

.dockerignore

@@ -1,6 +1,7 @@
 .git
 .github
 .pytest_cache
+.mypy_cache
 build
 dist
 docs

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,12 +1,21 @@
 # Overview
 
-# Related Issue / Discussion
+# Related issue / discussion
 
-# Additional Information
+# Additional information
 
-# Contributions and Licensing
+# Dependency policy (RFC2)
+
+- [ ] I have ensured that this PR meets [RFC2](https://pygeoapi.io/development/rfc/2) requirements
+
+# Updates to public demo
+
+- [ ] I have ensured that breaking changes to the [pygeoapi master demo server](https://github.com/geopython/demo.pygeoapi.io) have been addressed
+  - [ ] https://github.com/geopython/demo.pygeoapi.io/blob/master/services/pygeoapi_master/local.config.yml
+
+# Contributions and licensing
 
 (as per https://github.com/geopython/pygeoapi/blob/master/CONTRIBUTING.md#contributions-and-licensing)
 
-- [ ] I'd like to contribute [feature X|bugfix Y|docs|something else] to pygeoapi. I confirm that my contributions to pygeoapi will be compatible with the pygeoapi license guidelines at the time of contribution.
+- [ ] I'd like to contribute [feature X|bugfix Y|docs|something else] to pygeoapi. I confirm that my contributions to pygeoapi will be compatible with the pygeoapi license guidelines at the time of contribution
 - [ ] I have already previously agreed to the pygeoapi Contributions and Licensing Guidelines

.github/workflows/containers.yml

@@ -18,7 +18,7 @@ env:
 jobs:
   on-success:
     name: Build, Test and Push Docker Image to DockerHub
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
     permissions:
       packages: write
@@ -80,7 +80,7 @@ jobs:
           platforms: linux/arm64, linux/amd64
 
   on-failure:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: ${{ github.event.workflow_run.conclusion == 'failure' }}
     steps:
       - name: Print Test Fail

.github/workflows/main.yml

@@ -15,7 +15,7 @@ on:
 
 jobs:
   flake8_py3:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Setup Python
         uses: actions/setup-python@v1
@@ -35,35 +35,18 @@ jobs:
 
   main:
     needs: [flake8_py3]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         include:
           - python-version: 3.8
     env:
       PYGEOAPI_CONFIG: "$(pwd)/pygeoapi-config.yml"
 
-    services:
-      # Oracle service (label used to access the service container)
-      oracle:
-        # Docker Hub image (feel free to change the tag "latest" to any other available one)
-        image: gvenzl/oracle-xe:latest
-        # Provide passwords and other environment variables to container
-        env:
-          ORACLE_RANDOM_PASSWORD: true
-          APP_USER: geo_test
-          APP_USER_PASSWORD: geo_test
-        # Forward Oracle port
-        ports:
-          - 1521:1521
-        # Provide healthcheck script options for startup
-        options: >-
-          --health-cmd healthcheck.sh
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 10
-    
     steps:
+    - name: Chown user
+      run: |
+        sudo chown -R $USER:$USER $GITHUB_WORKSPACE
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
       name: Setup Python ${{ matrix.python-version }}
@@ -105,9 +88,13 @@ jobs:
       with:
         packages: gdal-bin libgdal-dev
         version: 3.0.4
+    - name: Install and run Oracle
+      run: |
+        docker run -d --name oracledb -e ORACLE_PWD=oracle -v ${{ github.workspace }}/tests/data/oracle/init-db:/opt/oracle/scripts/startup -p 1521:1521 container-registry.oracle.com/database/express:21.3.0-xe
     - name: Install requirements 📦
       run: |
         pip3 install -r requirements.txt
+        pip3 install -r requirements-admin.txt
         pip3 install -r requirements-starlette.txt
         pip3 install -r requirements-dev.txt
         pip3 install -r requirements-provider.txt
@@ -125,6 +112,7 @@ jobs:
         python3 tests/load_mongo_data.py tests/data/ne_110m_populated_places_simple.geojson
         gunzip < tests/data/hotosm_bdi_waterways.sql.gz | psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test
         psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_data.sql
+        docker ps
         python3 tests/load_oracle_data.py
     - name: run unit tests ⚙️
       env:
@@ -164,3 +152,44 @@ jobs:
       if: ${{ failure() }}
       run: |
         pip3 list -v
+
+  admin:
+    needs: [flake8_py3]
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        include:
+          - python-version: 3.8
+    env:
+      PYGEOAPI_CONFIG: "tests/pygeoapi-test-config-admin.yml"
+      PYGEOAPI_OPENAPI: "tests/pygeoapi-test-openapi-admin.yml"
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      name: Setup Python ${{ matrix.python-version }}
+      with:
+        python-version: ${{ matrix.python-version }}
+    - uses: awalsh128/cache-apt-pkgs-action@latest
+      with:
+        packages: gunicorn python3-gevent
+        version: 1.0
+    - name: Install requirements 📦
+      run: |
+        pip3 install -r requirements.txt
+        pip3 install -r requirements-dev.txt
+        pip3 install -r requirements-admin.txt
+        python3 setup.py install
+    - name: Run pygeoapi with admin API ⚙️
+      run: |
+        pygeoapi openapi generate ${PYGEOAPI_CONFIG} --output-file ${PYGEOAPI_OPENAPI}
+        gunicorn --bind 0.0.0.0:5000 \
+                 --reload \
+                 --reload-extra-file ${PYGEOAPI_CONFIG} \
+                 pygeoapi.flask_app:APP &
+    - name: run integration tests ⚙️
+      run: |
+        pytest tests/test_admin_api.py
+    - name: failed tests 🚩
+      if: ${{ failure() }}
+      run: |
+        pip3 list -v

.github/workflows/vulnerabilities.yml

@@ -0,0 +1,46 @@
+name: Check vulnerabilities
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+      - '!**.md'
+  release:
+    types:
+      - released
+
+jobs:
+
+  vulnerabilities:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: .
+    steps:
+    - name: Checkout pygeoapi
+      uses: actions/checkout@v4
+    - name: Scan vulnerabilities with trivy
+      uses: aquasecurity/trivy-action@master
+      with:
+        scan-type: fs
+        exit-code: 1
+        ignore-unfixed: true
+        severity: CRITICAL,HIGH
+        scanners: vuln,misconfig,secret
+        scan-ref: .
+    - name: Build locally the image from Dockerfile
+      run: |
+        docker buildx build -t ${{ github.repository }}:${{ github.sha }} --platform linux/amd64 --no-cache -f Dockerfile .
+    - name: Scan locally built Docker image for vulnerabilities with trivy
+      uses: aquasecurity/trivy-action@master
+      with:
+        scan-type: image
+        exit-code: 1
+        ignore-unfixed: true
+        severity: CRITICAL,HIGH
+        vuln-type: os,library
+        image-ref: '${{ github.repository }}:${{ github.sha }}'

.gitignore

@@ -125,3 +125,8 @@ pygeoapi/db.sqlite3
 
 # Pycharm project files
 .idea
+
+# ES data folder
+.pygeoapi/docker/examples/elastic/ES/data
+.pygeoapi/docker/examples/mvt-elastic/ES/data
+.pygeoapi/docker/examples/mvt-tippecanoe/ES/data

.trivyignore

@@ -0,0 +1,2 @@
+# Accept the risk
+DS002  # Dockerfile with non-root user

Dockerfile

@@ -9,6 +9,7 @@
 # Copyright (c) 2019 Just van den Broecke
 # Copyright (c) 2020 Francesco Bartoli
 # Copyright (c) 2021 Angelos Tzotsos
+# Copyright (c) 2023 Bernhard Mallinger
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
@@ -33,7 +34,7 @@
 #
 # =================================================================
 
-FROM ubuntu:jammy 
+FROM ubuntu:jammy-20231211.1
 
 LABEL maintainer="Just van den Broecke <justb4@gmail.com>"
 
@@ -64,6 +65,7 @@ ARG ADD_DEB_PACKAGES="\
     python3-elasticsearch \
     python3-fiona \
     python3-gdal \
+    python3-jsonpatch \
     python3-netcdf4 \
     python3-pandas \
     python3-psycopg2 \
@@ -100,12 +102,10 @@ ENV TZ=${TZ} \
     ${ADD_DEB_PACKAGES}"
 
 WORKDIR /pygeoapi
-ADD . /pygeoapi
 
 # Install operating system dependencies
 RUN \
     apt-get update -y \
-    && apt-get upgrade -y \
     && apt-get --no-install-recommends install -y ${DEB_PACKAGES} ${DEB_BUILD_DEPS}  \
     && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \
     && echo "For ${TZ} date=$(date)" && echo "Locale=$(locale)"  \
@@ -119,21 +119,30 @@ RUN \
     && unzip ./SCHEMAS_OPENGIS_NET.zip "ogcapi/*" -d /schemas.opengis.net \
     && rm -f ./SCHEMAS_OPENGIS_NET.zip \
 
-    # Install remaining pygeoapi deps
-    && pip3 install -r requirements-docker.txt \
-
-    # Install pygeoapi
-    && pip3 install -e . \
-
-    # Set default config and entrypoint for Docker Image
-    && cp /pygeoapi/docker/default.config.yml /pygeoapi/local.config.yml \
-    && cp /pygeoapi/docker/entrypoint.sh /entrypoint.sh  \
-
     # Cleanup TODO: remove unused Locales and TZs
+    # NOTE: this tries to remove gcc, but the actual package gcc-11 can't be
+    #       removed because python3-scipy depends on python3-pythran which
+    #       depends on g++
     && apt-get remove --purge -y gcc ${DEB_BUILD_DEPS} \
     && apt-get clean \
     && apt autoremove -y  \
     && rm -rf /var/lib/apt/lists/*
 
+ADD requirements-docker.txt requirements-admin.txt /pygeoapi/
+# Install remaining pygeoapi deps
+RUN python3 -m pip install --no-cache-dir -r requirements-docker.txt \
+    && python3 -m pip install --no-cache-dir -r requirements-admin.txt
+
+
+ADD . /pygeoapi
+
+ # Install pygeoapi
+RUN python3 -m pip install --no-cache-dir -e . 
+
+RUN \ 
+    # Set default config and entrypoint for Docker Image
+    cp /pygeoapi/docker/default.config.yml /pygeoapi/local.config.yml \
+    && cp /pygeoapi/docker/entrypoint.sh /entrypoint.sh 
+
 ENTRYPOINT ["/entrypoint.sh"]
 

LICENSE.md

@@ -1,6 +1,6 @@
 # The MIT License (MIT)
 
-Copyright © 2018-2023 pygeoapi development team
+Copyright © 2018-2024 pygeoapi development team
 
 * * *
 

README.md

@@ -3,6 +3,7 @@
 [![DOI](https://zenodo.org/badge/121585259.svg)](https://zenodo.org/badge/latestdoi/121585259)
 [![Build](https://github.com/geopython/pygeoapi/actions/workflows/main.yml/badge.svg)](https://github.com/geopython/pygeoapi/actions/workflows/main.yml)
 [![Docker](https://github.com/geopython/pygeoapi/actions/workflows/containers.yml/badge.svg)](https://github.com/geopython/pygeoapi/actions/workflows/containers.yml)
+[![Vulnerabilities](https://github.com/geopython/pygeoapi/actions/workflows/vulnerabilities.yml/badge.svg)](https://github.com/geopython/pygeoapi/actions/workflows/vulnerabilities.yml)
 
 [pygeoapi](https://pygeoapi.io) is a Python server implementation of the [OGC API](https://ogcapi.ogc.org) suite of standards. The project emerged as part of the next generation OGC API efforts in 2018 and provides the capability for organizations to deploy a RESTful OGC API endpoint using OpenAPI, GeoJSON, and HTML. pygeoapi is [open source](https://opensource.org/) and released under an [MIT license](https://github.com/geopython/pygeoapi/blob/master/LICENSE.md).