Merge pull request #29 from lucernae/django

Add CI with helm charts install

.envrc

@@ -1,8 +1,14 @@
 use_nix
 
+export KUBECONFIG=`pwd`/kubeconfig.yaml
+export PROJECT_ROOT=`pwd`
+export CT_CONFIG=`pwd`/ct.yaml
+
 source_env_if_exists .local.envrc
 
-dotenv_if_exists .local.env
+dotenv_if_exists .env .local.env
+
+watch_file .local.envrc .local.env
 
 if [[ -z "$(which docker)" ]]; then
 	echo "You need to install docker"

.github/workflows/chart-lint.yaml

@@ -24,16 +24,65 @@ jobs:
       - name: Set up chart-testing
         uses: helm/chart-testing-action@v2.0.1
 
+      - name: Set up nix
+        uses: cachix/install-nix-action@v12
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+
+      - name: Set up nix store cache
+        uses: actions/cache@v2
+        with:
+          path: /tmp/nix-closure
+          # Build inputs are:
+          # - shell.nix
+          # - nix/ directory
+          key: nix-store-${{ hashFiles('default.nix', 'nix') }}
+          restore-keys: |
+            nix-store-${{ hashFiles('default.nix', 'nix') }}-
+            nix-store-
+
+      - name: Loads nix derivation cache
+        id: load-cache
+        run: |
+          if [[ -f /tmp/nix-closure/nix-shell.closure ]]; then
+            nix-store --import --option require-sigs false < /tmp/nix-closure/nix-shell.closure
+            echo "::set-output name=use-cache::true"
+          fi
+      - run: |
+          nix-env -f '<nixpkgs>' -iA pkgs.direnv
+          nix-build
+          nix-shell --run "direnv allow"
+      - name: Store nix derivation cache
+        run: |
+          DERIVATION=$(readlink result)
+          echo $DERIVATION
+          mkdir -p /tmp/nix-closure
+          nix-store --export $(nix-store -qR ${DERIVATION}) > /tmp/nix-closure/nix-shell.closure
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        shell: direnv exec . bash -eo pipefail {0}
+        run: |
+          changed=$(ct list-changed)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
       - name: Run chart-testing (lint)
-        run: ct lint --config ct.yaml
-
-# Disable this actions until we have a good alternative to check helm install
-#      - name: Create kind cluster
-#        uses: helm/kind-action@v1.0.0
-#        # Only build a kind cluster if there are chart changes to test.
-#        if: steps.lint.outputs.changed == 'true'
-#
-#      - name: Run chart-testing (install)
-#        uses: helm/chart-testing-action@v1.0.0
-#        with:
-#          command: install
+        id: lint
+        shell: direnv exec . bash -eo pipefail {0}
+        run: ct lint
+
+      - name: Create kind cluster
+        uses: helm/kind-action@v1.0.0
+        with:
+          node_image: "kindest/node:v1.19.1"
+        # Only build a kind cluster if there are chart changes to test.
+        if: steps.list-changed.outputs.changed == 'true'
+
+      - name: Run chart-testing (install)
+        if: steps.list-changed.outputs.changed == 'true'
+        shell: direnv exec . bash -eo pipefail {0}
+        run: |
+          unset KUBECONFIG
+          ct install

.gitignore

@@ -2,7 +2,18 @@
 .idea
 .python-version
 Chart.lock
-.direnv
 
+# direnv related
+.direnv
 .local.envrc
 .local.env
+
+# k8s related
+kubeconfig.yaml
+
+# nix related
+result
+local-shell.nix
+
+# chart-testing related
+ct.override.yaml

DEVELOPMENT-README.md

@@ -9,9 +9,16 @@ same, we use Nix to pin the exact dependencies. We use Direnv to provide the
 local settings hooks
 
 - Install [Docker](https://docs.docker.com/get-docker/)
-- Install [Direnv](https://direnv.net/)
 - Install [Nix](https://nixos.org/manual/nix/stable/#sect-multi-user-installation) using multi-user installation if possible
-- Install nix-direnv
+- Install [Direnv](https://direnv.net/)
+
+Or use Nix to install direnv
+
+```bash
+nix-env -f '<nixpkgs>' -iA direnv
+```
+
+- Install nix-direnv (Nix optimized plugin for direnv)
 
 ```bash
 nix-env -f '<nixpkgs>' -iA nix-direnv
@@ -35,13 +42,13 @@ Direnv files, `.envrc` works cascadingly. In the root directory, there is a
 If you need to override these values, create extra file called `.local.envrc` or `.env`.
 This will be executed to override `.envrc` in that directory.
 
-Nix-shell function `shell.nix` is used to express the dependencies we need to 
+Nix function/derivation `default.nix` is used to express the dependencies we need to 
 setup our tools. It is also called by Direnv when we enter root directory of 
-this project.
+this project. You can customize your own Nix shell by creating a `local-shell.nix`
 
 To setup the environment at first time, run this in the root project directory:
 
-```
+```bash
 direnv allow
 ```
 
@@ -55,12 +62,61 @@ Chart testing works by comparing diff changes in the repo. So we need to provide
 which branch we are working on against. Normally you have `origin` and `upstream` 
 in a git based workflow. If not, then you need to set your `upstream` remote accordingly.
 
+Overriding any Chart testing option can be done by setting environment variable 
+via your `.local.env` file. To see which option are available, check:
+https://github.com/helm/chart-testing. If that is not enough, 
+set `CT_CONFIG=<absolute path to this project path>/ct.override.yaml` and then 
+set chart testing config file in `ct.override.yaml`. Example use case is when 
+you want to repeatedly test specific chart in local.
+
 Running
 
-```
+```bash
 ct lint
 ```
 
 Will do a lint by comparing the current branch with the current `main` branch 
-in local repo. To override this behaviour, refer to the chart-testing docs and 
-do override necessary in the `.envrc` or `.env` directory
+in local repo.
+
+# Generating chart documentations
+
+Chart is autodocumented using default `values.yaml` and helm-docs.
+
+Execute `helm-docs` in chart subdirectory to generate the docs.
+
+Modification can be done in respective `README.md.gotmpl` file.
+
+# Chart testing
+
+In order to test the chart, you need a k8s distro running.
+
+If you already have a sandbox k8s cluster, put the kubeconfig file in `kubeconfig.yaml` 
+in the project root directory. Alternatively, specify `KUBECONFIG` environment 
+in your `.local.env` file in the project directory
+
+If you don't have a sandbox k8s cluster, create one in your local machine.
+It's recommended to use [KIND](https://kind.sigs.k8s.io/) because it's lightweight.
+
+Create cluster by running
+
+```bash
+kind create cluster --config=kind.config.yaml
+```
+
+You can replace `kind.config.yaml` file with any valid kind config file.
+Once KIND is running, you can extract the kube config file like this:
+
+```bash
+kind export kubeconfig --kubeconfig kubeconfig.yaml
+```
+
+Once you got that set up,
+
+Running
+
+```bash
+ct install
+```
+
+Will install changed charts into your target cluster.
+`ct` will also run `helm test` if the chart have test hook.

chart_schema.yaml

@@ -0,0 +1,37 @@
+name: str()
+home: str(required=False)
+version: str()
+apiVersion: str()
+appVersion: any(str(), num(), required=False)
+description: str()
+keywords: list(str(), required=False)
+sources: list(str(), required=False)
+maintainers: list(include('maintainer'), required=False)
+dependencies: list(include('dependency'), required=False)
+icon: str(required=False)
+engine: str(required=False)
+condition: str(required=False)
+tags: str(required=False)
+deprecated: bool(required=False)
+kubeVersion: str(required=False)
+annotations: map(str(), str(), required=False)
+type: str(required=False)
+---
+maintainer:
+  name: str()
+  email: str(required=False)
+  url: str(required=False)
+---
+dependency:
+  name: str()
+  version: str()
+  repository: str()
+  condition: str(required=False)
+  tags: list(str(), required=False)
+  enabled: bool(required=False)
+  import-values: any(list(str()), list(include('import-value')), required=False)
+  alias: str(required=False)
+---
+import-value:
+  child: str()
+  parent: str()

charts/common/v1.0.0/templates/_common.pod.yaml

@@ -0,0 +1,9 @@
+{{- define "common.pod.tpl" -}}
+apiVersion: v1
+kind: Pod
+{{ include "common.metadata.tpl" . }}
+{{- end -}}
+
+{{- define "common.pod" -}}
+{{ include "common.util.merge" (append . "common.pod.tpl") }}
+{{- end -}}

charts/django/v0.2.1/.envrc

@@ -0,0 +1 @@
+source_env_if_exists ../../../.envrc

charts/django/v0.2.1/ci/wagtail-values.yaml

@@ -121,6 +121,8 @@ extraConfigMap: |
         # We do manual copy because they put /code/bakerydemo/media/images as volumes
         # So it overshadows our volumes
         cp -rf /code/bakerydemo/media/* {{ .Values.global.mediaRoot }}
+        cp -rf {{ .Values.global.mediaRoot }}/original_images/* {{ .Values.global.mediaRoot }}/images
+        chown -R 1000:2000 {{ .Values.global.mediaRoot }}
     fi
 
     # Doing collectstatic
@@ -179,3 +181,22 @@ extraPodSpec: |
         - name: static-dir
           mountPath: {{ .Values.global.staticRoot }}
           subPath: {{ .Values.persistence.staticDir.subPath }}
+
+test:
+  django:
+    containers: |
+      - name: {{ .Release.Name }}-test
+        image: curlimages/curl
+        command:
+          - /bin/sh
+          - -ec
+          - |
+            set -eux
+            # check basic url
+            curl -v -H 'Host: {{ .Values.global.siteName }}' {{ template "common.fullname" . }}
+            # check about page
+            curl -v -H 'Host: {{ .Values.global.siteName }}' {{ template "common.fullname" . }}/about/
+            # check breads page
+            curl -v -H 'Host: {{ .Values.global.siteName }}' {{ template "common.fullname" . }}/breads/
+            # check admin page
+            curl -v -H 'Host: {{ .Values.global.siteName }}' {{ template "common.fullname" . }}/admin/

charts/django/v0.2.1/templates/secrets.yaml

@@ -21,12 +21,14 @@ data:
   {{- end }}
 
   # Django database connection URL
-  {{- if not .Values.global.databasePassword.valueFrom.secretKeyRef.name }}
+  {{- if (and (not .Values.global.databasePassword.valueFrom.secretKeyRef.name) (not .Values.global.databaseURL.value) ) }}
   {{- $databasePassword := include "common.secretValue" .Values.global.databasePassword }}
   {{ .Values.global.databasePassword.valueFrom.secretKeyRef.key }}: {{ $databasePassword | b64enc | quote }}
   postgresql-password: {{ $databasePassword | b64enc | quote }}
   {{- $databaseHost := (default .Values.global.databaseHost (printf "%s-postgis-headless" .Release.Name) ) }}
   database-url: {{ printf "postgres://%s:%s@%s:%d/%s" .Values.global.databaseUsername $databasePassword $databaseHost (int .Values.global.databasePort) .Values.global.databaseName | b64enc | quote }}
+  {{- else if .Values.global.databaseURL.value }}
+  database-url: {{ .Values.global.databaseURL.value | b64enc | quote }}
   {{- end }}
 
 {{- end}}

charts/django/v0.2.1/templates/tests/test-django.yaml

@@ -0,0 +1,23 @@
+{{- include "common.pod" (list . "django.test.pod") -}}
+{{- define "django.test.pod" -}}
+metadata:
+  name: {{ template "common.fullname" . }}-test
+  annotations:
+    helm.sh/hook: test
+spec:
+  restartPolicy: Never
+  containers:
+    {{- if not .Values.test.django.containers }}
+    - name: {{ .Release.Name }}-test
+      image: curlimages/curl
+      command:
+        - /bin/sh
+        - -ec
+        - |
+          curl -v -H 'Host: {{ .Values.global.siteName }}' {{ template "common.fullname" . }}
+    {{- else }}
+    {{- with .Values.test.django.containers }}
+    {{- tpl . $ | nindent 4 }}
+    {{- end }}
+    {{- end }}
+{{- end -}}

charts/django/v0.2.1/templates/tests/test-postgres-connection.yaml

@@ -0,0 +1,29 @@
+{{- define "django.test.postgres.pod" -}}
+metadata:
+  name: {{ template "common.fullname" . }}-postgres-test
+  annotations:
+    helm.sh/hook: test
+spec:
+  restartPolicy: Never
+  containers:
+    - name: {{ .Release.Name }}-postgres-test
+      # The Django image itself must contains binaries that can connect
+      # to the said database
+      image: {{ template "common.image" . }}
+      env:
+        - name: DATABASE_URL
+          {{- $param := dict "Value" .Values.global.databaseURL "Context" . -}}
+          {{ include "common.secretFrom" $param | nindent 10 }}
+      command:
+        - /bin/sh
+        - -ec
+        - |
+          psql ${DATABASE_URL} -c "\dt+" > /tmp/test-output.txt
+          cat /tmp/test-output.txt
+{{- end -}}
+
+{{- if .Values.postgis.enabled -}}
+
+{{- include "common.pod" (list . "django.test.postgres.pod") -}}
+
+{{- end -}}

charts/django/v0.2.1/values.yaml

@@ -90,6 +90,15 @@ global:
   # -- (string) Storage class name used to provision PV
   storageClassName:
 
+  # -- (string) Use this if you want to specify the full
+  # Database URL connection
+  databaseURL:
+    value:
+    valueFrom:
+      secretKeyRef:
+        name:
+        key: database-url
+
 # -- (tpl/array) Define this for extra Django environment variables
 # @notationType -- tpl
 extraPodEnv: |
@@ -255,3 +264,9 @@ persistence:
     accessModes:
       - ReadWriteOnce
     annotations: {}
+
+test:
+  django:
+    # -- (tpl/array) List of containers override for testing
+    # @notationType -- (tpl)
+    containers: |