[DRAFT] add CI tool for docker health check and initial ci workflow

.github/workflows/install-docker.yaml

@@ -0,0 +1,90 @@
+name: Test App Install
+
+on:
+  pull_request: {}
+
+jobs:
+  changed-files:
+    name: Get changed apps
+    runs-on: ubuntu-latest
+    outputs:
+      changed-apps: ${{ steps.changed-apps.outputs.changed-apps }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Get changed files
+        id: changed-files-json
+        uses: tj-actions/changed-files@v44
+        with:
+          json: true
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.22.2"
+      - name: Get changed apps
+        id: changed-apps
+        env:
+          CHANGED_FILES: ${{ steps.changed-files-json.outputs.all_changed_files }}
+        run: |
+          echo $CHANGED_FILES
+          out=$(go run ./tools/get-changed-apps/cmd/main.go)
+          echo "changed-apps=${out}" >> $GITHUB_OUTPUT
+          echo $out
+
+  test-apps:
+    name: Test apps
+    needs: changed-files
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        app: ${{ fromJson(needs.changed-files.outputs.changed-apps) }}
+    steps:
+      - name: Environment Information
+        run: |
+          echo "====== Docker Info ======"
+          docker info
+          echo "========================="
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.22.2"
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Dependencies
+        shell: bash
+        run: |
+          pip install -r requirements.txt
+      - name: Install Go Dependencies
+        run: |
+          cd tools/docker-health-check
+          go mod download
+          go build -o ../../check_health cmd/main.go
+          chmod +x ../../check_health
+      - name: Run app
+        shell: bash
+        run: |
+          for values_file in ix-dev/${{ matrix.app }}/ci/*.yaml; do
+            echo "Testing [$values_file]"
+            sudo ./main.py $values_file ix-dev/${{ matrix.app }} > docker-compose.yaml
+
+            # Print the parsed, by compose, template
+            # (as it will also remove empty lines)
+            docker compose -f docker-compose.yaml config
+
+            PROJECT_NAME="$(openssl rand -hex 12)"
+
+            docker compose -p $PROJECT_NAME -f docker-compose.yaml up -d
+
+            ./check_health --project $PROJECT_NAME --files docker-compose.yaml
+
+            docker compose -p $PROJECT_NAME -f docker-compose.yaml down --remove-orphans --volumes
+            docker compose -p $PROJECT_NAME -f docker-compose.yaml rm --force --stop --volumes
+
+            # Clean up the test directory used by the app
+            echo "Cleaning up /mnt/test directory"
+            sudo rm -r /mnt/test || echo "Failed to clean up /mnt/test"
+          done

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+ix-dev/**/rendered

.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+  "files.associations": {
+    "*.yaml": "jinja-yaml"
+  },
+  "[jinja-yaml]": {
+    "editor.defaultFormatter": null,
+    "editor.formatOnSave": false,
+    "editor.formatOnPaste": false
+  }
+}

copy_lib.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Copy the library to the ix-dev directory
+train=$1
+app_name=$2
+
+if [ -z "$train" ]; then
+  echo "Usage: $0 <train> <app_name>"
+  exit 1
+fi
+
+if [ -z "$app_name" ]; then
+  echo "Usage: $0 <app_name>"
+  exit 1
+fi
+
+app_path="ix-dev/$train/$app_name"
+
+if [ ! -d "$app_path" ]; then
+  echo "App [$app_path] does not exist"
+  exit 1
+fi
+
+# pick the latest version
+lib=$(ls -d library/* | sort -V | tail -n 1)
+lib_version=$(basename $lib)
+rm -rf "$app_path/templates/library/base_v$(sed 's/\./_/g' <<<$lib_version)"
+mkdir -p "$app_path/templates/library/$lib_version"
+cp -r $lib/* "$app_path/templates/library/$lib_version"
+mv "$app_path/templates/library/$lib_version" "$app_path/templates/library/base_v$(sed 's/\./_/g' <<<$lib_version)"

cspell.config.yaml

@@ -0,0 +1,4 @@
+words:
+  - CPUS
+  - Healtcheck
+  - isready

library/1.0.0/snippets.py

@@ -0,0 +1,74 @@
+from . import utils
+
+def health_check(test = "", interval = 10, timeout = 10, retries = 5, start_period = 30):
+  if not test:
+    utils.throw_error("Healtcheck: [test] must be set")
+
+  return {
+    "test": test,
+    "interval": f'{interval}s',
+    "timeout": f'{timeout}s',
+    "retries": retries,
+    "start_period": f'{start_period}s'
+  }
+
+def curl_test(url):
+  return f"curl --silent --fail {url}"
+
+def pg_test(user, db, host = "127.0.0.1", port = 5432):
+  if not user:
+    utils.throw_error("Postgres container: [user] must be set")
+
+  if not db:
+    utils.throw_error("Postgres container: [db] must be set")
+
+  return f"pg_isready -h {host} -p {port} -d {db} -U {user}"
+
+def postgres_uid():
+  return 999
+def postgres_gid():
+  return 999
+def postgres_run_as():
+  return f"{postgres_uid()}:{postgres_gid()}"
+
+def postgres_environment(user, password, db):
+  if not user:
+    utils.throw_error("Postgres container: [user] must be set")
+
+  if not password:
+    utils.throw_error("Postgres container: [password] must be set")
+
+  if not db:
+    utils.throw_error("Postgres container: [db] must be set")
+
+  return {
+    "POSTGRES_USER": user,
+    "POSTGRES_PASSWORD": password,
+    "POSTGRES_DB": db
+  }
+
+def get_default_limits():
+  return {
+    "cpus": "2.0",
+    "memory": "4gb"
+  }
+
+def get_limits(data):
+  limits = get_default_limits()
+
+  if not data:
+    return limits
+
+  limits.update({
+    "cpus": str(data.get("limits", limits['cpus']).get("cpus", limits['cpus'])),
+    "memory": data.get("limits", limits['memory']).get("memory", limits['memory'])
+  })
+
+  return limits
+
+def resources(data = {}):
+  return {
+    "resources": {
+      "limits": get_limits(data)
+    },
+  }

library/1.0.0/utils.py

@@ -0,0 +1,85 @@
+from . import validations
+import secrets
+import yaml
+import sys
+import os
+
+class TemplateException(Exception):
+  pass
+
+def throw_error(message: str) -> None:
+  # When throwing a known error, hide the traceback
+  # This is because the error is also shown in the UI
+  # and having a traceback makes it hard for user to read
+  sys.tracebacklimit = 0
+  raise TemplateException(message)
+
+def get_yaml_opts():
+    return {
+        'default_flow_style': False,
+        'sort_keys': False,
+        'indent': 2,
+    }
+
+def to_yaml(data):
+    return yaml.dump(data, **get_yaml_opts())
+
+def secure_string(length):
+    return secrets.token_urlsafe(length)
+
+
+# TODO: maybe extend this with ACLs API?!
+def host_path_with_perms(data: dict, root: dict, perms: dict) -> str:
+    if not data.get('type', ''):
+        throw_error("Host Path Configuration: Type must be set")
+
+    path = ''
+    if data['type'] == 'host_path':
+        path = process_host_path(data, root, perms)
+    elif data['type'] == 'ix_volume':
+        path = process_ix_volume(data, root)
+    else:
+        throw_error(f"Type [{data['type']}] is not supported")
+
+    # FIXME: only use this on CI and not on prod
+    os.makedirs(path, exist_ok=True)
+
+    # FIXME: only do such things if user agreed or if its ixVolumes
+    if perms.get('user') and perms.get('group'):
+        # Set permissions
+        os.chown(path, int(perms['user']), int(perms['group']))
+
+    return validations.func_validate_path(path)
+
+def process_ix_volume(data: dict, root: dict) -> dict:
+    path = ''
+    if not data.get('ix_volume_config', {}):
+      throw_error("IX Volume Configuration: [ix_volume_config] must be set")
+
+    if not data['ix_volume_config'].get('dataset_name', ''):
+      throw_error("IX Volume Configuration: [ix_volume_config.dataset_name] must be set")
+
+    if not root.get('ixVolumes', []):
+      throw_error("IX Volume Configuration: [ixVolumes] must be set")
+
+    for item in root['ixVolumes']:
+      if not item.get('hostPath', ''):
+        throw_error("IX Volume Configuration: [ixVolumes] item must contain [hostPath]")
+
+      if item['hostPath'].split('/')[-1] == data['ix_volume_config']['dataset_name']:
+        path = item['hostPath']
+        break
+
+    if not path:
+      throw_error(f"IX Volume Configuration: [ixVolumes] does not contain dataset with name [{data['ix_volume_config']['dataset_name']}]")
+
+    return path
+
+def process_host_path(data: dict) -> str:
+    if not data.get('host_path_config', {}):
+      throw_error("Host Path Configuration: [host_path_config] must be set")
+
+    if not data['host_path_config'].get('path', ''):
+      throw_error("Host Path Configuration: [host_path_config.path] must be set")
+
+    return data['host_path_config']['path']

library/1.0.0/validations.py

@@ -0,0 +1,43 @@
+from . import utils
+import re
+
+def is_email(email):
+  return re.match(r"[^@]+@[^@]+\.[^@]+", email)
+
+def must_be_email(email):
+  if not is_email(email):
+    utils.throw_error("Value must be an email address")
+
+  return email
+
+def must_be_length(value, length):
+  if len(value) < length:
+    utils.throw_error(f"Value must be {length} characters long")
+
+  return value
+
+def is_password_secure(password):
+  checks=[
+    lambda p: len(p) >= 8,
+    lambda p: re.search("[a-z]", p),
+    lambda p: re.search("[A-Z]", p),
+    lambda p: re.search("[0-9]", p),
+    lambda p: re.search("[!@#$%^&*(),.?\":{}|<>]", p)
+  ]
+
+  return all(c(password) for c in checks)
+
+def must_be_password_secure(password):
+    if not is_password_secure(password):
+        utils.throw_error("Password must contain at least 8 characters, 1 uppercase letter, 1 lowercase letter, 1 number, and 1 special character")
+
+    return password
+
+def validate_path(path):
+    if not path:
+        utils.throw_error("Path must be set")
+
+    if not path.startswith("/"):
+        utils.throw_error(f"Path [{path}] must start with [/]")
+
+    return ''

main.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+import importlib
+import inspect
+import yaml
+import sys
+import os
+from jinja2 import Environment, FileSystemLoader
+
+if len(sys.argv) < 3:
+  raise ValueError("path must be set")
+
+values_path = sys.argv[1]
+app_path = sys.argv[2]
+
+
+if not values_path:
+  raise ValueError("values path must be set")
+
+if not app_path:
+  raise ValueError("app path must be set")
+
+if not os.path.exists(values_path):
+  raise ValueError(f"values path [{values_path}] does not exist")
+
+if not os.path.exists(app_path):
+  raise ValueError(f"app path [{app_path}] does not exist")
+
+env = Environment(
+    loader=FileSystemLoader(f"{app_path}/templates"),
+)
+
+items = [
+  importlib.import_module("library.common.validations"),
+  importlib.import_module("library.common.snippets"),
+  importlib.import_module("library.common.utils")
+]
+
+for item in items:
+  for name, func in inspect.getmembers(item, inspect.isfunction):
+    if name.startswith("filter_"):
+      env.filters[name.replace("filter_", "")] = func
+    if name.startswith("func_"):
+      env.globals.update({name.replace("func_", ""): func})
+
+template = env.get_template("docker-compose.yaml.j2")
+
+with open(values_path) as f:
+  mock = yaml.safe_load(f)
+
+print(template.render(data=mock))

requirements.txt

@@ -0,0 +1,2 @@
+Jinja2
+pyyaml