Merge pull request #15 from truenas/library-hashes

Generate hashes of different library versions and validate them

apps_validation/validation/json_schema_utils.py

@@ -33,10 +33,20 @@
             'type': 'string',
             'pattern': '[0-9]+.[0-9]+.[0-9]+',
         },
+        'lib_version_hash': {'type': 'string'},
     },
     'required': [
         'name', 'train', 'version',
     ],
+    'if': {
+        'properties': {
+            'lib_version': {'type': 'string'},
+        },
+        'required': ['lib_version'],
+    },
+    'then': {
+        'required': ['lib_version_hash'],
+    },
 }
 APP_MIGRATION_SCHEMA = {
     'type': 'array',
@@ -73,6 +83,14 @@
         ],
     },
 }
+BASE_LIBRARIES_JSON_SCHEMA = {
+    'type': 'object',
+    'patternProperties': {
+        '[0-9]+.[0-9]+.[0-9]+': {
+            'type': 'string',
+        },
+    },
+}
 CATALOG_JSON_SCHEMA = {
     'type': 'object',
     'patternProperties': {

apps_validation/validation/validate_app_version.py

@@ -8,6 +8,7 @@
 
 from apps_validation.exceptions import ValidationErrors
 from catalog_reader.app_utils import get_app_basic_details
+from catalog_reader.hash_utils import get_hash_of_directory
 from catalog_reader.names import get_base_library_dir_name_from_version
 from catalog_reader.questions_util import CUSTOM_PORTALS_KEY
 
@@ -67,6 +68,8 @@ def validate_catalog_item_version(
             )
         elif not base_lib_dir.is_dir():
             verrors.add(f'{schema}.lib_version', f'{base_lib_dir!r} is not a directory')
+        elif get_hash_of_directory(str(base_lib_dir)) != app_basic_details['lib_version_hash']:
+            verrors.add(f'{schema}.lib_version', 'Library version hash does not match with the actual library version')
 
     questions_path = os.path.join(version_path, 'questions.yaml')
     if os.path.exists(questions_path):

apps_validation/validation/validate_catalog.py

@@ -11,6 +11,7 @@
 from .json_schema_utils import CATALOG_JSON_SCHEMA
 from .validate_app_rename_migrations import validate_migrations
 from .validate_app import validate_catalog_item
+from .validate_library import validate_base_libraries
 from .validate_recommended_apps import validate_recommended_apps_file
 from .validate_train import get_train_items, validate_train_structure
 
@@ -41,6 +42,8 @@ def validate_catalog(catalog_path: str):
 
     verrors.check()
 
+    validate_base_libraries(catalog_path, verrors)
+
     for method, params in (
         (validate_recommended_apps_file, (catalog_path,)),
         (validate_migrations, (os.path.join(catalog_path, 'migrations'),)),

apps_validation/validation/validate_dev_directory.py

@@ -13,6 +13,7 @@
 
 from .app_version import validate_app_version_file
 from .validate_app_version import validate_catalog_item_version
+from .validate_library import validate_base_libraries
 
 
 def validate_dev_directory_structure(catalog_path: str, to_check_apps: dict) -> None:
@@ -28,6 +29,8 @@ def validate_dev_directory_structure(catalog_path: str, to_check_apps: dict) ->
         validate_train(
             catalog_path, os.path.join(dev_directory, train_name), f'dev.{train_name}', to_check_apps[train_name]
         )
+
+    validate_base_libraries(catalog_path, verrors)
     verrors.check()
 
 

apps_validation/validation/validate_library.py

@@ -0,0 +1,52 @@
+import pathlib
+
+from jsonschema import validate as json_schema_validate, ValidationError as JsonValidationError
+
+from apps_validation.exceptions import ValidationErrors
+from catalog_reader.library import get_library_hashes, get_hashes_of_base_lib_versions, RE_VERSION
+from catalog_reader.names import get_library_path, get_library_hashes_path
+
+from .json_schema_utils import BASE_LIBRARIES_JSON_SCHEMA
+
+
+def validate_base_libraries(catalog_path: str, verrors: ValidationErrors) -> None:
+    library_path = get_library_path(catalog_path)
+    library_path_obj = pathlib.Path(library_path)
+    if not library_path_obj.exists():
+        return
+
+    found_libs = False
+    for entry in filter(lambda e: e.is_dir(), library_path_obj.iterdir()):
+        if RE_VERSION.match(entry.name):
+            found_libs = True
+        else:
+            verrors.add(
+                f'library.{entry.name}', 'Library version folder should conform to semantic versioning i.e 1.0.0'
+            )
+
+    if found_libs and not pathlib.Path(get_library_hashes_path(library_path)).exists():
+        verrors.add('library', 'Library hashes file is missing')
+
+    verrors.check()
+
+    get_local_hashes_contents = get_library_hashes(library_path)
+    try:
+        json_schema_validate(get_local_hashes_contents, BASE_LIBRARIES_JSON_SCHEMA)
+    except JsonValidationError as e:
+        verrors.add('library', f'Invalid format specified for library hashes: {e}')
+
+    verrors.check()
+
+    try:
+        hashes = get_hashes_of_base_lib_versions(catalog_path)
+    except Exception as e:
+        verrors.add('library', f'Error while generating hashes for library versions: {e}')
+    else:
+        if hashes != get_local_hashes_contents:
+            verrors.add(
+                'library',
+                'Generated hashes for library versions do not match with the existing '
+                'hashes file and need to be updated'
+            )
+
+    verrors.check()

catalog_reader/app_utils.py

@@ -47,7 +47,7 @@ def get_app_basic_details(app_path: str) -> dict:
     with contextlib.suppress(FileNotFoundError, yaml.YAMLError, KeyError):
         with open(os.path.join(app_path, 'app.yaml'), 'r') as f:
             app_config = yaml.safe_load(f.read())
-        return {'lib_version': app_config.get('lib_version')} | {
+        return {k: app_config.get(k) for k in ('lib_version', 'lib_version_hash')} | {
             k: app_config[k] for k in ('name', 'train', 'version')
         }
 

catalog_reader/hash_utils.py

@@ -0,0 +1,12 @@
+import subprocess
+
+
+def get_hash_of_directory(directory: str) -> str:
+    """
+    This returns sha256sum of the directory
+    """
+    cp = subprocess.run(
+        f'find {directory} -type f -exec sha256sum {{}} + | sort | awk \'{{print $1}}\' | sha256sum',
+        capture_output=True, check=True, shell=True,
+    )
+    return cp.stdout.decode().split()[0]

catalog_reader/library.py

@@ -0,0 +1,33 @@
+import contextlib
+import os
+import pathlib
+import re
+import yaml
+
+from .hash_utils import get_hash_of_directory
+from .names import get_library_path, get_library_hashes_path
+
+
+RE_VERSION = re.compile(r'^\d+.\d+\.\d+$')
+
+
+def get_library_hashes(library_path: str) -> dict:
+    """
+    This reads from library hashes file and returns the hashes
+    """
+    with contextlib.suppress(FileNotFoundError, yaml.YAMLError):
+        with open(get_library_hashes_path(library_path), 'r') as f:
+            return yaml.safe_load(f.read())
+
+
+def get_hashes_of_base_lib_versions(catalog_path: str) -> dict:
+    library_path = get_library_path(catalog_path)
+    library_dir = pathlib.Path(library_path)
+    hashes = {}
+    for lib_entry in library_dir.iterdir():
+        if not lib_entry.is_dir() or not RE_VERSION.match(lib_entry.name):
+            continue
+
+        hashes[lib_entry.name] = get_hash_of_directory(os.path.join(library_path, lib_entry.name))
+
+    return hashes

catalog_reader/names.py

@@ -1,6 +1,8 @@
+import os
 import typing
 
 
+LIBRARY_HASHES_FILENAME = 'hashes.yaml'
 RECOMMENDED_APPS_FILENAME = 'recommended_apps.yaml'
 TO_KEEP_VERSIONS = 'to_keep_versions.yaml'
 UPGRADE_STRATEGY_FILENAME = 'upgrade_strategy'
@@ -12,3 +14,11 @@ def get_app_library_dir_name_from_version(version: str) -> str:
 
 def get_base_library_dir_name_from_version(version: typing.Optional[str]) -> str:
     return f'base_v{version.replace(".", "_")}' if version else ''
+
+
+def get_library_hashes_path(library_path: str) -> str:
+    return os.path.join(library_path, LIBRARY_HASHES_FILENAME)
+
+
+def get_library_path(catalog_path: str) -> str:
+    return os.path.join(catalog_path, 'library')

catalog_reader/scripts/apps_hashes.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+import argparse
+import os
+import pathlib
+import ruamel.yaml
+import shutil
+import yaml
+
+from apps_validation.exceptions import CatalogDoesNotExist, ValidationErrors
+from catalog_reader.dev_directory import get_ci_development_directory
+from catalog_reader.library import get_hashes_of_base_lib_versions
+from catalog_reader.names import get_library_path, get_library_hashes_path, get_base_library_dir_name_from_version
+
+
+YAML = ruamel.yaml.YAML()
+YAML.indent(mapping=2, sequence=4, offset=2)
+
+
+def update_catalog_hashes(catalog_path: str) -> None:
+    if not os.path.exists(catalog_path):
+        raise CatalogDoesNotExist(catalog_path)
+
+    verrors = ValidationErrors()
+    library_dir = pathlib.Path(get_library_path(catalog_path))
+    if not library_dir.exists():
+        verrors.add('library', 'Library directory is missing')
+
+    verrors.check()
+
+    hashes = get_hashes_of_base_lib_versions(catalog_path)
+    hashes_file_path = get_library_hashes_path(get_library_path(catalog_path))
+    with open(hashes_file_path, 'w') as f:
+        yaml.safe_dump(hashes, f)
+
+    print(f'[\033[92mOK\x1B[0m]\tGenerated hashes for library versions at {hashes_file_path!r}')
+
+    dev_directory = pathlib.Path(get_ci_development_directory(catalog_path))
+    if not dev_directory.is_dir():
+        return
+    elif not hashes:
+        print('[\033[92mOK\x1B[0m]\tNo hashes found for library versions, skipping updating apps hashes')
+        return
+
+    for train_dir in dev_directory.iterdir():
+        if not train_dir.is_dir():
+            continue
+
+        for app_dir in train_dir.iterdir():
+            if not app_dir.is_dir():
+                continue
+
+            app_metadata_file = app_dir / 'app.yaml'
+            if not app_metadata_file.is_file():
+                continue
+
+            with open(str(app_metadata_file), 'r') as f:
+                app_config = YAML.load(f)
+
+            if (lib_version := app_config.get('lib_version')) and lib_version not in hashes:
+                print(
+                    f'[\033[93mWARN\x1B[0m]\tLibrary version {lib_version!r} not found in hashes, '
+                    f'skipping updating {app_dir.name!r} in {train_dir.name} train'
+                )
+                continue
+
+            base_lib_name = get_base_library_dir_name_from_version(lib_version)
+            app_lib_dir = app_dir / 'templates/library'
+            app_lib_dir.mkdir(exist_ok=True, parents=True)
+            app_base_lib_dir = app_lib_dir / base_lib_name
+            shutil.rmtree(app_base_lib_dir.as_posix(), ignore_errors=True)
+
+            catalog_base_lib_dir_path = os.path.join(library_dir.as_posix(), lib_version)
+            shutil.copytree(catalog_base_lib_dir_path, app_base_lib_dir.as_posix())
+
+            app_config['lib_version_hash'] = hashes[lib_version]
+            with open(str(app_metadata_file), 'w') as f:
+                YAML.dump(app_config, f)
+
+            print(f'[\033[92mOK\x1B[0m]\tUpdated library hash for {app_dir.name!r} in {train_dir.name}')
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--path', help='Specify path of TrueNAS catalog')
+
+    args = parser.parse_args()
+    if not args.path:
+        parser.print_help()
+    else:
+        update_catalog_hashes(args.path)
+
+
+if __name__ == '__main__':
+    main()

requirements.txt

@@ -3,4 +3,5 @@ jinja2
 jsonschema==4.10.3
 markdown
 pyyaml
+ruamel.yaml
 semantic_version

setup.py

@@ -22,6 +22,7 @@
     platforms='any',
     entry_points={
         'console_scripts': [
+            'apps_catalog_hash_generate = catalog_reader.scripts.apps_hashes:main',
             'apps_catalog_update = apps_validation.scripts.catalog_update:main',
             'apps_catalog_validate = apps_validation.scripts.catalog_validate:main',
             'apps_dev_charts_validate = apps_validation.scripts.dev_apps_validate:main',  # TODO: Remove apps_prefix