Merge pull request #2 from truenas/dev-dir-changes

Add changes to allow validating dev directory structure

.github/workflows/docker_image.yml

@@ -0,0 +1,28 @@
+name: build_image
+
+on:
+  push:
+    branches:
+      - 'master_blocked'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          push: true
+          tags: ixsystems/apps_validation:latest
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

Dockerfile

@@ -0,0 +1,22 @@
+FROM ghcr.io/truenas/middleware:master
+
+RUN /usr/bin/install-dev-tools
+
+RUN apt-get install -y \
+      debhelper-compat \
+      dh-python \
+      python3-dev \
+      python3-setuptools \
+      devscripts \
+      python3-jsonschema \
+      python3-semantic-version \
+      python3-yaml
+
+ENV PYTHONUNBUFFERED 1
+ENV WORK_DIR /app
+RUN mkdir -p ${WORK_DIR}
+WORKDIR ${WORK_DIR}
+
+ADD . ${WORK_DIR}/
+RUN pip install --break-system-packages -r requirements.txt
+RUN pip install --break-system-packages -U .

apps_validation/catalog_reader/dev_directory.py

@@ -7,6 +7,7 @@
 from .names import TO_KEEP_VERSIONS
 
 
+DEV_DIRECTORY_RELATIVE_PATH = 'ix-dev'
 OPTIONAL_METADATA_FILES = ['upgrade_info.json', 'upgrade_strategy', TO_KEEP_VERSIONS]
 REQUIRED_METADATA_FILES = ['item.yaml']
 REQUIRED_VERSIONS_JSON_SCHEMA = {
@@ -25,7 +26,7 @@ def get_app_version(app_path: str) -> str:
 
 
 def get_ci_development_directory(catalog_path: str) -> str:
-    return os.path.join(catalog_path, 'ix-dev')
+    return os.path.join(catalog_path, DEV_DIRECTORY_RELATIVE_PATH)
 
 
 def version_has_been_bumped(app_path: str, new_version: str) -> bool:

apps_validation/ci/git.py

@@ -0,0 +1,42 @@
+import os
+import subprocess
+from collections import defaultdict
+
+from apps_validation.catalog_reader.dev_directory import (
+    DEV_DIRECTORY_RELATIVE_PATH, get_ci_development_directory, OPTIONAL_METADATA_FILES
+)
+from apps_validation.catalog_reader.train_utils import is_train_valid
+from apps_validation.exceptions import CatalogDoesNotExist
+
+
+def get_changed_apps(catalog_path: str, base_branch: str = 'master') -> dict:
+    if not os.path.exists(catalog_path):
+        raise CatalogDoesNotExist(catalog_path)
+
+    cp = subprocess.run(
+        ['git', '-C', catalog_path, '--no-pager', 'diff', '--name-only', base_branch],
+        capture_output=True, check=True,
+    )
+    dev_directory_path = get_ci_development_directory(catalog_path)
+    to_check_apps = defaultdict(list)
+    for file_path in filter(
+        lambda path: path and path.startswith(f'{DEV_DIRECTORY_RELATIVE_PATH}/'),
+        map(str.strip, cp.stdout.decode().split('\n'))
+    ):
+        dev_dir_relative_path = file_path.strip(f'{DEV_DIRECTORY_RELATIVE_PATH}/')
+        train_name = dev_dir_relative_path.split('/', 1)[0]
+        if not is_train_valid(train_name, os.path.join(dev_directory_path, train_name)):
+            continue
+
+        app_name = dev_dir_relative_path.split('/')[1]
+        base_name = os.path.basename(file_path)
+
+        if base_name in OPTIONAL_METADATA_FILES:
+            continue
+        if not os.path.isdir(os.path.join(dev_directory_path, train_name, app_name)):
+            continue
+
+        if app_name not in to_check_apps[train_name]:
+            to_check_apps[train_name].append(app_name)
+
+    return to_check_apps

apps_validation/scripts/dev_apps_validate.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+import argparse
+
+from apps_validation.ci.git import get_changed_apps
+from apps_validation.validation.validate_dev_directory import validate_dev_directory_structure
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(help='sub-command help', dest='action')
+
+    parser_setup = subparsers.add_parser(
+        'validate', help='Validate TrueNAS dev catalog items'
+    )
+    parser_setup.add_argument('--path', help='Specify path of TrueNAS dev catalog', required=True)
+    parser_setup.add_argument(
+        '--base_branch', help='Specify base branch to find changed catalog items', default='master'
+    )
+
+    args = parser.parse_args()
+    if args.action == 'validate':
+        validate_dev_directory_structure(args.path, get_changed_apps(args.path, args.base_branch))
+    else:
+        parser.print_help()
+
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -12,6 +12,8 @@
     license='GNU3',
     platforms='any',
     entry_points={
-        'console_scripts': [],
+        'console_scripts': [
+            'dev_charts_validate = apps_validation.scripts.dev_apps_validate:main',
+        ],
     },
 )