Merge pull request #1 from DCOR-dev/dcor_develop

Develop a CKAN based DCOR image with custom developed extensions

.env.example

@@ -29,7 +29,7 @@ USE_HTTPS_FOR_DEV=false
 # CKAN core
 CKAN_VERSION=2.10.0
 CKAN_SITE_ID=default
-CKAN_SITE_URL=https://localhost:8443
+CKAN_SITE_URL=http://localhost:8443
 CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
 # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
 CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
@@ -65,6 +65,16 @@ CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://ckan:5000
 NGINX_PORT=80
 NGINX_SSLPORT=443
 
+# S3/MINIO settings
+CKAN_STORAGE_BACKEND=s3
+CKANEXT__S3FILESTORE__HOST_NAME=http://minio:9000
+CKANEXT__S3FILESTORE__AWS_BUCKET_NAME=ckan
+CKANEXT__S3FILESTORE__AWS_ACCESS_KEY_ID=MINIOACCESSKEY
+CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=MINIOSECRETKEY
+CKANEXT__S3FILESTORE__REGION_NAME=us-east-1
+CKANEXT__S3FILESTORE__SIGNATURE_VERSION=s3v4
+
+
 # Extensions
 CKAN__PLUGINS="image_view text_view datatables_view datastore datapusher envvars"
 CKAN__HARVEST__MQ__TYPE=redis

.github/workflows/build_and_push_dcor.yml

@@ -0,0 +1,52 @@
+name: Build and Push DCOR-CKAN Docker Image
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - '*.*.*'  # This triggers the workflow on tags that follow the X.Y.Z format
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Build and Push NGINX Image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./nginx
+          file: ./nginx/Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/dcor-ckan-docker-nginx:${{ github.ref_name  }}
+            ${{ secrets.DOCKER_USERNAME }}/dcor-ckan-docker-nginx:latest
+
+      - name: Build and Push PostgreSQL Image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./postgresql
+          file: ./postgresql/Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/dcor-ckan-docker-postgresql:${{ github.ref_name  }}
+            ${{ secrets.DOCKER_USERNAME }}/dcor-ckan-docker-postgresql:latest
+
+      - name: Build and Push CKAN Image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./ckan
+          file: ./ckan/Dockerfile.dev  # Build dcor image from dev
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/dcor-ckan-docker-ckan:${{ github.ref_name  }}
+            ${{ secrets.DOCKER_USERNAME }}/dcor-ckan-docker-ckan:latest

.gitignore

@@ -1,6 +1,7 @@
 # generic
 .DS_Store
 .vagrant
+.idea
 
 # code & config
 _service-provider/*

CHANGELOG

@@ -0,0 +1,3 @@
+0.1.0
+ - initial version of dcor-ckan image
+ - build and push dcor-ckan image to dockerhub

ckan/Dockerfile.dev

@@ -1,38 +1,31 @@
 FROM ckan/ckan-dev:2.11
 
-# Install any extensions needed by your CKAN instance
-# - Make sure to add the plugins to CKAN__PLUGINS in the .env file
-# - Also make sure all provide all extra configuration options, either by:
-#   * Adding them to the .env file (check the ckanext-envvars syntax for env vars), or
-#   * Adding extra configuration scripts to /docker-entrypoint.d folder) to update
-#      the CKAN config file (ckan.ini) with the `ckan config-tool` command
-#
-# See README > Extending the base images for more details
-#
-# For instance:
-#
-### XLoader ###
-#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@master#egg=ckanext-xloader' && \ 
-#    pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \
-#    pip3 install -U requests[security]
-
-### Harvester ###
-#RUN pip3 install -e 'git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest' && \
-#    pip3 install -r ${APP_DIR}/src/ckanext-harvest/pip-requirements.txt
-# will also require gather_consumer and fetch_consumer processes running (please see https://github.com/ckan/ckanext-harvest)
-
-### Scheming ###
-#RUN  pip3 install -e 'git+https://github.com/ckan/ckanext-scheming.git@master#egg=ckanext-scheming'
-
-### Pages ###
-#RUN  pip3 install -e git+https://github.com/ckan/ckanext-pages.git#egg=ckanext-pages
-
-### DCAT ###
-#RUN  pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v0.0.6#egg=ckanext-dcat && \
-#     pip3 install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v0.0.6/requirements.txt
-
-# Clone the extension(s) your are writing for your own project in the `src` folder
-# to get them mounted in this image at runtime
+# Change to root user
+USER root
+# Install vim for editing files
+RUN apt-get update && apt-get install -y vim
+
+# dcor_control installation takes place in ".local" and cached in ".cache" dirs
+# so ckan need to access these dirs.
+RUN mkdir /srv/app/.local && chown -R ckan:ckan-sys /srv/app/.local
+RUN mkdir /srv/app/.cache && chown -R ckan:ckan-sys /srv/app/.cache
+
+# Add python bin (dcor_control is installed here) dir to PATH
+ENV PATH="/srv/app/.local/bin:${PATH}"
+
+# dcor stores configuration keys here
+RUN mkdir /srv/app/.config && \
+    chown -R ckan:ckan-sys /srv/app/.config
+
+# Create the directory (if it doesn't already exist)
+RUN chown -R ckan:ckan-sys /var/log && \
+    chmod -R 775 /var/log
+
+RUN mkdir -p /var/lib/ckan/storage/uploads/admin && \
+    chown -R ckan:ckan-sys /var/lib/ckan/storage/uploads
+
+RUN mkdir -p /var/lib/ckan/i18n && \
+    chown -R ckan:ckan-sys /var/lib/ckan/i18n
 
 # Copy custom initialization scripts
 COPY --chown=ckan-sys:ckan-sys docker-entrypoint.d/* /docker-entrypoint.d/

ckan/docker-entrypoint.d/02_setup_dcor.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+pip3 install dcor_control
+dcor inspect --assume-yes

docker-compose.dcor.dev.yml

@@ -0,0 +1,100 @@
+volumes:
+  ckan_storage:
+  pg_data:
+  solr_data:
+  pip_cache:
+  site_packages:
+  vscode_server:
+  minio_data:
+
+services:
+  dcor-ckan-dev:
+    build:
+      context: ckan/
+      dockerfile: Dockerfile.dev
+      args:
+        - TZ=${TZ}
+    env_file:
+      - .env
+    links:
+      - db
+      - solr
+      - redis
+      - minio
+    # This is defined as a docker resource in `dcor_control`
+    hostname: dcor-dev-docker
+    ports:
+      - "0.0.0.0:${CKAN_PORT_HOST}:5000"
+    volumes:
+      - ckan_storage:/var/lib/ckan
+      - ./src:/srv/app/src_extensions
+      - pip_cache:/root/.cache/pip
+      - site_packages:/usr/lib/python3.10/site-packages
+      - vscode_server:/root/.vscode-server
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
+
+  datapusher:
+    image: ckan/ckan-base-datapusher:${DATAPUSHER_VERSION}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8800"]
+
+  db:
+    build:
+      context: postgresql/
+    environment:
+      - POSTGRES_USER
+      - POSTGRES_PASSWORD
+      - POSTGRES_DB
+      - CKAN_DB_USER
+      - CKAN_DB_PASSWORD
+      - CKAN_DB
+      - DATASTORE_READONLY_USER
+      - DATASTORE_READONLY_PASSWORD
+      - DATASTORE_DB
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
+
+  solr:
+    image: ckan/ckan-solr:${SOLR_IMAGE_VERSION}
+    volumes:
+      - solr_data:/var/solr
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:8983/solr/"]
+
+  redis:
+    image: redis:${REDIS_VERSION}
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "redis-cli", "-e", "QUIT"]
+
+  minio:
+    image: minio/minio:latest
+    restart: unless-stopped
+    environment:
+      - MINIO_ROOT_USER
+      - MINIO_ROOT_PASSWORD
+    command: minio server --console-address ":9001" /data
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - minio_data:/data
+
+  mc:
+    image: minio/mc:RELEASE.2020-08-08T02-33-58Z
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc config host rm local;
+      /usr/bin/mc config host add --api s3v4 local http://minio:9000 minioadmin minioadmin;
+      /usr/bin/mc mb local/ckan/;
+      /usr/bin/mc policy set download local/ckan/storage;
+      "