hypervisor: add ability to unpack hypervisor specialization package

ebcl/tools/hypervisor/config_gen.py

@@ -1,13 +1,18 @@
 import argparse
 import logging
 import os
+
 from pathlib import Path
+from tempfile import TemporaryDirectory
 
 import jinja2
 import yaml
 
 from ebcl.common import init_logging, log_exception
+from ebcl.common.config import Config
 from ebcl.common.files import resolve_file
+from ebcl.common.version import VersionDepends
+from ebcl.tools.hypervisor.model_gen import ConfigError
 
 from .schema_loader import BaseModel, FileReadProtocol, Schema, merge_dict
 
@@ -26,8 +31,7 @@ def load(self, config_file: str, conf_dir: Path) -> dict:
         }
         while config["base"]:
             base_name = config["base"].pop()
-            base_path = resolve_file(
-                file=base_name, relative_base_dir=str(conf_dir))
+            base_path = resolve_file(file=base_name, relative_base_dir=str(conf_dir))
             old = config
             config = self._load_file(base_path)
             merge_dict(config, old)
@@ -91,6 +95,64 @@ def create_files(self) -> None:
                 outpath.write_text(template.read_text("utf-8"))
 
 
+class SpecializationUnpacker:
+    """
+    Unpacks a debian package with hypervisor specialization
+
+    The property directory returns the path to the files on the local filesystem.
+    They are removed when this object is deleted.
+
+    The path is looked up automatically searching for schema.yaml. It can also be
+    specified in the constructor (path).
+    """
+    _tmp_dir: TemporaryDirectory
+    _config_dir: Path
+
+    def __init__(self, package_name: str, config_file: Path, path: str | None = None) -> None:
+        self._tmp_dir = TemporaryDirectory()
+        self._config_dir = Path(self._tmp_dir.name)
+
+        config = Config(str(config_file), self._tmp_dir.name)
+
+        package = config.proxy.find_package(
+            VersionDepends(package_name, None, None, None, config.arch)
+        )
+        if not package:
+            raise ConfigError(f"Cannot find package {package_name}")
+        package = config.proxy.download_package(config.arch, package)
+        if not package:  # pragma: no cover (should never happen)
+            raise ConfigError(f"Cannot download package {package_name}")
+
+        print(package.local_file)
+        print(package.extract(self._tmp_dir.name, None, use_sudo=False))
+
+        os.system(f"find {self._tmp_dir.name}")
+
+        if path:
+            self._config_dir = self._config_dir / path
+        else:
+            self._config_dir = self.find_schema()
+
+    def find_schema(self) -> Path:
+        """Find a single instance of schema.yaml in the archive"""
+        directories: list[Path] = []
+        for root, _, files in os.walk(self._tmp_dir.name):
+            if "schema.yaml" in files:
+                directories.append(Path(root))
+
+        if len(directories) != 1:
+            raise ConfigError(f"Expected exactly one schema.yaml in specialization package, found {len(directories)}.")
+        return directories[0]
+
+    def __del__(self) -> None:
+        self._tmp_dir.cleanup()
+
+    @property
+    def directory(self) -> Path:
+        """Path to the extension directory on the local filesystem"""
+        return self._config_dir
+
+
 @log_exception(call_exit=True)
 def main() -> None:
     """ Main entrypoint of EBcL hypervisor generator. """
@@ -102,17 +164,26 @@ def main() -> None:
 
     parser = argparse.ArgumentParser(
         description='Create the config files for the hypervisor')
-    parser.add_argument('-s', '--specialization', type=str,
+    parser.add_argument('-s', '--specialization', type=Path,
                         help='Path to hypervisor specialization directory')
-    parser.add_argument('config_file', type=str,
+    parser.add_argument('-p', '--specialization-package', type=str,
+                        help='Name of a debian package that contains the hypervisor specialization')
+    parser.add_argument('--specialization-path', type=str,
+                        help='Path to specialization in package')
+    parser.add_argument('-r', '--repo-config', type=Path,
+                        help='Path to a config file with a repository containing the hypervisor specialization')
+    parser.add_argument('config_file', type=Path,
                         help='Path to the YAML configuration file')
-    parser.add_argument('output', type=str,
+    parser.add_argument('output', type=Path,
                         help='Path to the output directory')
     args = parser.parse_args()
 
-    if args.specialization:
-        args.specialization = Path(args.specialization)
-    generator = HvFileGenerator(Path(args.config_file), Path(args.output), args.specialization)
+    if args.specialization_package:
+        if not args.repo_config or not args.repo_config.exists():
+            parser.error("If a SPECIALIZATION_PACKAGE is specified a REPO_CONFIG must be specified as well")
+        unpacker = SpecializationUnpacker(args.specialization_package, args.repo_config, args.specialization_path)
+        args.specialization = unpacker.directory
+    generator = HvFileGenerator(args.config_file, args.output, args.specialization)
     generator.create_files()
 
 

tests/data/example_packages/Makefile

@@ -23,17 +23,6 @@ $$(deb).deb: DEB=$(NAME)_$(subst :,+,$(1))_$(2)
 endef
 
 $(foreach arch,$(archs),$(foreach version,$(versions),$(eval $(call gen_targets,$(version),$(arch)))))
-$(info $(debs))
-
-package: DEBIAN/changelog DEBIAN/control
-	dpkg-deb -b . $(NAME)_$(VERSION).deb
-
-DEBIAN:
-	mkdir -p $@
-
-DEBIAN/%: %.in | DEBIAN
-	sed 's/@@NAME@@/$(NAME)/; s/@@VERSION@@/$(VERSION)/; s/@@ARCH@@/$(ARCH)/;' $< > $@
-
 
 clean:
 	rm -rf tmp

tests/hypervisor/data/test_packages/.gitignore

@@ -0,0 +1 @@
+tmp

tests/hypervisor/data/test_packages/InRelease

@@ -0,0 +1,8 @@
+Origin: Test Repo
+Label: TestRepo
+Architectures: amd64 arm64
+Description: Test repo
+Date: Wed Dec 11 12:00:00 2024
+SHA256:
+ 3f77a280600be6e2cfb825741278847cc440f156c9012525a3006028b33f4f93 1209 Packages
+ 1b34a7b9d81aa35bb27f6898530988e563c0b064b97b35f069d84742819f8e33 628 Packages.xz

tests/hypervisor/data/test_packages/Makefile

@@ -0,0 +1,65 @@
+names := test-empty test-simple test-multiple test-ebclfsa
+
+files_test-empty := files-empty
+arch_test-empty := arm64
+
+files_test-simple := files-ok
+arch_test-simple := arm64
+
+files_test-multiple := files-ok files-ok2
+arch_test-multiple := arm64
+
+# Reuse files from the example and include too much (expected and config) on purpose!
+files_test-ebclfsa := ../examples/qemu_ebclfsa
+arch_test-ebclfsa := arm64
+
+debs := $(addsuffix .deb,$(names))
+all: $(debs) Packages.xz InRelease
+
+
+define gen_targets
+deb := $(1)
+tmp/$$(deb)/DEBIAN:
+	mkdir -p $$@
+tmp/$$(deb)/DEBIAN/%: %.in | tmp/$$(deb)/DEBIAN
+	sed 's/@@NAME@@/$(1)/; s/@@VERSION@@/$$(VERSION)/; s/@@ARCH@@/$$(ARCH)/;' $$< > $$@
+$$(deb).deb: tmp/$$(deb)/DEBIAN/control
+	$$(foreach f,$(files_$(1)),cp -r $$(f)/* tmp/$$(DEB);)
+	dpkg-deb -b tmp/$$(DEB) $$@
+	rm -rf tmp/$$(DEB)
+$$(deb).deb: VERSION=1.0
+$$(deb).deb: ARCH=$(arch_$(1))
+$$(deb).deb: DEB=$(1)
+endef
+
+#$(foreach name,$(names),$(info $(call gen_targets,$(name))))
+$(foreach name,$(names),$(eval $(call gen_targets,$(name))))
+
+
+
+define IN_RELEASE_HEAD
+Origin: Test Repo
+Label: TestRepo
+Architectures: amd64 arm64
+Description: Test repo
+Date: Wed Dec 11 12:00:00 2024
+SHA256:
+endef
+export IN_RELEASE_HEAD
+
+Packages: $(debs)
+	dpkg-scanpackages -m . /dev/null > Packages
+	sed -i 's~./~~g' Packages
+
+Packages.xz: Packages
+	xz -c $^ > $@
+
+InRelease:
+	echo "$$IN_RELEASE_HEAD" > InRelease
+	echo " $$(sha256sum Packages | cut -d' ' -f1) $$(wc -c Packages)" >> InRelease
+	echo " $$(sha256sum Packages.xz | cut -d' ' -f1) $$(wc -c Packages.xz)" >> InRelease
+
+clean:
+	rm -rf tmp
+	rm -rf *.deb
+	rm -f Packages* InRelease

tests/hypervisor/data/test_packages/Packages

@@ -0,0 +1,48 @@
+Package: test-ebclfsa
+Version: 1.0
+Architecture: arm64
+Maintainer: nobody
+Filename: test-ebclfsa.deb
+Size: 3404
+MD5sum: f3d0fe80d4f93893cacb89b9ff7f4127
+SHA1: d455184b02b14bcd7d6674cc2b3d00a82c1cc759
+SHA256: 2848affa33fe5a1b56fb59b24da5e1172be10d577b06ff02521e5fa7d97f7685
+Description:
+ Example package
+
+Package: test-empty
+Version: 1.0
+Architecture: arm64
+Maintainer: nobody
+Filename: test-empty.deb
+Size: 488
+MD5sum: 2bee39e8c94278abbfafd5fe0a7d1653
+SHA1: 6b029df5dea0bbe02dbe5c7aa27155dc9fb66113
+SHA256: f3902ffe162954ab3ae3483efda97fadd6b9f48c9e67b2b77e208008c1eefbaf
+Description:
+ Example package
+
+Package: test-multiple
+Version: 1.0
+Architecture: arm64
+Maintainer: nobody
+Filename: test-multiple.deb
+Size: 546
+MD5sum: a2a503e734bb180d8e2fe0b7d6a5109a
+SHA1: 65b01ecd9ded1525ac0b0ce9b101f709a8591a2e
+SHA256: 1d6b9a3a0c6f2f6db5b44c7bc7033621a59d70270765e5386cb0a553733b0688
+Description:
+ Example package
+
+Package: test-simple
+Version: 1.0
+Architecture: arm64
+Maintainer: nobody
+Filename: test-simple.deb
+Size: 518
+MD5sum: 061515d128a6513c2302cd34dfa440a3
+SHA1: a9f1b0360f61994698c04e660fbe150df3c906d6
+SHA256: 3595fc09a69b84d8c4a7dcee5a0a3476bc7aab66178c64e2c3089f2b66ca9fca
+Description:
+ Example package
+

tests/hypervisor/data/test_packages/changelog.in

@@ -0,0 +1,3 @@
+@@NAME@@ (@@VERSION@@) UNRELEASED; urgency=medium
+    nothing
+ -- nobody <nobody@example.com>  Mon, 06 Mar 2017 06:52:08 +0100

tests/hypervisor/data/test_packages/control.in

@@ -0,0 +1,6 @@
+Package: @@NAME@@
+Architecture: @@ARCH@@
+Maintainer: nobody
+Version: @@VERSION@@
+Description:
+ Example package

tests/hypervisor/data/test_packages/files-ok/a/b/schema.yaml

@@ -0,0 +1 @@
+version: 1

tests/hypervisor/data/test_packages/files-ok2/c/d/schema.yaml

@@ -0,0 +1 @@
+version: 1