From 3496a7f8940a123ce99f38f8622231c09cf1f0d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Vehlow?= Date: Tue, 3 Dec 2024 13:19:47 +0100 Subject: [PATCH] add initial version of hypervisor tooling (#18) * deb: fix extraction of uncompressed data.tar In some debian packages the data.tar is not compressed and named only data.tar without any extension. * add initial version of hypervisor tooling Adds tooling to generated the hypervisor configuration (lua scripts) using a yaml configuration file and a yaml model of this configuration described in yaml and python. The reason for having the model "twice", is that the yaml definition of the model describes how the configuration is parsed into a python object model. The python model is required to do some postprocessing of the configuration, before jinja2 is used to generate the final configuration. The used hypervisor version can support additional features or require a different generated configuration. For that reason the configuration model can be extended by a configuration specialization bundled together with the hypervisor. --- ebcl/tools/hypervisor/__init__.py | 0 ebcl/tools/hypervisor/config_gen.py | 115 ++++++ ebcl/tools/hypervisor/data/io.cfg.j2 | 32 ++ ebcl/tools/hypervisor/data/model.schema.json | 45 +++ ebcl/tools/hypervisor/data/modules.list.j2 | 26 ++ ebcl/tools/hypervisor/data/schema.yaml | 134 +++++++ ebcl/tools/hypervisor/data/system.lua | 154 ++++++++ ebcl/tools/hypervisor/model.py | 373 +++++++++++++++++++ ebcl/tools/hypervisor/model_gen.py | 114 ++++++ ebcl/tools/hypervisor/schema_loader.py | 196 ++++++++++ pyproject.toml | 1 + 11 files changed, 1190 insertions(+) create mode 100644 ebcl/tools/hypervisor/__init__.py create mode 100644 ebcl/tools/hypervisor/config_gen.py create mode 100755 ebcl/tools/hypervisor/data/io.cfg.j2 create mode 100644 ebcl/tools/hypervisor/data/model.schema.json create mode 100755 ebcl/tools/hypervisor/data/modules.list.j2 create mode 100644 ebcl/tools/hypervisor/data/schema.yaml create mode 100755 ebcl/tools/hypervisor/data/system.lua create mode 100644 ebcl/tools/hypervisor/model.py create mode 100644 ebcl/tools/hypervisor/model_gen.py create mode 100644 ebcl/tools/hypervisor/schema_loader.py diff --git a/ebcl/tools/hypervisor/__init__.py b/ebcl/tools/hypervisor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ebcl/tools/hypervisor/config_gen.py b/ebcl/tools/hypervisor/config_gen.py new file mode 100644 index 0000000..f74bd81 --- /dev/null +++ b/ebcl/tools/hypervisor/config_gen.py @@ -0,0 +1,115 @@ +import argparse +import logging +import os +from pathlib import Path + +import jinja2 +import yaml + +from ebcl.common import init_logging, log_exception +from ebcl.common.files import resolve_file + +from .schema_loader import BaseModel, FileReadProtocol, Schema, merge_dict + + +class BaseResolver: + """ + Resolve bases defined in the yaml config. + """ + + def load(self, config_file: str, conf_dir: str) -> dict: + """ + Load config_file and all of its bases. + """ + config = { + "base": [config_file] + } + while config["base"]: + base_name = config["base"].pop(0) + base_path = resolve_file( + file=base_name, relative_base_dir=conf_dir) + old = config + config = self._load_file(base_path) + merge_dict(config, old) + return config + + def _load_file(self, filename: str) -> dict: + with open(filename, "r", encoding="utf-8") as f: + return yaml.load(f, yaml.Loader) + + +class HvFileGenerator: + """ + Hypervisor configuration file generator + """ + config: BaseModel + schema: Schema + output_path: Path + + def __init__(self, file: str, output_path: str, specialization: str | None = None) -> None: + """ Parse the yaml config file. + Args: + config_file (Path): Path to the yaml config file. + """ + self.output_path = Path(output_path) + + """ Load yaml configuration. """ + config_file = Path(file) + + config = BaseResolver().load(config_file.name, str(config_file.parent)) + del config["base"] + + self.schema = Schema(specialization and Path(specialization) or None) + self.config = self.schema.parse_config(config) + + def _render_template(self, outpath: Path, template: FileReadProtocol) -> None: + """Render a template to target""" + template_obj = jinja2.Template(template.read_text("utf-8"), trim_blocks=True) + + with outpath.open("w", encoding="utf-8") as f: + f.write(template_obj.render(config=self.config)) + + def create_files(self) -> None: + """ + Create three Files of HV + init.ned, io.cfg and modules.list + """ + self.output_path.mkdir(exist_ok=True) + + for template in self.schema.templates: + base, ext = os.path.splitext(template.name) + if ext == ".j2": + logging.info("Rendering %s", base) + outpath = self.output_path / base + self._render_template(outpath, template) + else: + logging.info("Creating %s", template) + outpath = self.output_path / template.name + outpath.write_text(template.read_text("utf-8")) + + +@log_exception(call_exit=True) +def main() -> None: + """ Main entrypoint of EBcL hypervisor generator. """ + init_logging() + + logging.info('\n===================\n' + 'EBcL Hypervisor Config File Generator\n' + '===================\n') + + parser = argparse.ArgumentParser( + description='Create the config files for the hypervisor') + parser.add_argument('-s', '--specialization', type=str, + help='Path to hypervisor specialization directory') + parser.add_argument('config_file', type=str, + help='Path to the YAML configuration file') + parser.add_argument('output', type=str, + help='Path to the output directory') + args = parser.parse_args() + + generator = HvFileGenerator(args.config_file, args.output, args.specialization) + generator.create_files() + + +if __name__ == "__main__": + main() diff --git a/ebcl/tools/hypervisor/data/io.cfg.j2 b/ebcl/tools/hypervisor/data/io.cfg.j2 new file mode 100755 index 0000000..47f0ad7 --- /dev/null +++ b/ebcl/tools/hypervisor/data/io.cfg.j2 @@ -0,0 +1,32 @@ +local Res = Io.Res +local Hw = Io.Hw + +local hw = Io.system_bus(); + +Io.Dt.add_children(hw, function() +{% for vbus in config.vbus %} +{% for dev in vbus.devices %} + {{dev.name}} = Hw.Device(function() +{% if dev.compatible %} + compatible = {"{{dev.compatible}}"} +{% endif %} +{%for mmio in dev.mmios %} + Resource.reg{{loop.index0}} = Res.mmio({{"0x%x"|format(mmio.address)}}, {{"0x%x"|format(mmio.address + mmio.size - 1)}}{% if mmio.cached %}, Io.Resource.F_cached_mem | Io.Resource.F_prefetchable{% endif %}) +{% endfor %} +{%for irq in dev.irqs %} + Resource.irq{{loop.index0}} = Res.irq({{irq.irq + irq.offset}}, Io.Resource.Irq_type_{{irq.is_edge and "raising_edge" or "level_high"}}) +{% endfor %} + end) +{% endfor %} +{% endfor %} +end) + +Io.add_vbusses({ +{% for vbus in config.vbus %} + {{vbus.name}} = Io.Vi.System_bus(function() +{% for dev in vbus.devices %} + {{dev.name}} = wrap(hw.{{dev.name}}) +{% endfor %} + end), +{% endfor %} +}) diff --git a/ebcl/tools/hypervisor/data/model.schema.json b/ebcl/tools/hypervisor/data/model.schema.json new file mode 100644 index 0000000..79e4956 --- /dev/null +++ b/ebcl/tools/hypervisor/data/model.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://linux.elektrobit.com.com/model.schema.json", + "title": "Hypervisor Configuration Model", + "description": "Hypervisor configuration model", + "type": "object", + "additionalProperties": false, + "default": {}, + "properties": { + "version": { + "type": "integer" + }, + "classes": { + "type": "object", + "default": {}, + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": {"type": "string"}, + "default": {"type": ["string", "number", "boolean", "array"]}, + "aggregate": { + "enum": ["None", "list"], + "default": "None" + }, + "optional": {"type": "boolean", "default": false}, + "enum_values": {"type": "array", "items": {"type": "string"}} + } + } + } + }, + "root": { + "type": "string", + "optional": false + }, + "templates": { + "type": "array", + "items": { + "type": "string" + } + } + } +} diff --git a/ebcl/tools/hypervisor/data/modules.list.j2 b/ebcl/tools/hypervisor/data/modules.list.j2 new file mode 100755 index 0000000..e6d2778 --- /dev/null +++ b/ebcl/tools/hypervisor/data/modules.list.j2 @@ -0,0 +1,26 @@ +default-kernel fiasco -serial_esc + +entry hv-image +roottask moe rom/init.ned + +module l4re +module ned +module cons +module io +{% if config.vms %} +module uvmm +{% endif %} +{% if config.vio_block %} +module virtio-block +{% endif %} +{% if config.vnets %} +module l4vio_net_p2p +{% endif %} + +module init.ned +module io.cfg +module system.lua + +{% for module in config.modules %} +module {{module}} +{% endfor %} diff --git a/ebcl/tools/hypervisor/data/schema.yaml b/ebcl/tools/hypervisor/data/schema.yaml new file mode 100644 index 0000000..15d1f16 --- /dev/null +++ b/ebcl/tools/hypervisor/data/schema.yaml @@ -0,0 +1,134 @@ +# yaml-language-server: $schema=model.schema.json + +version: 1 + +classes: + HVConfig: + vbus: + type: VBus + aggregate: list + default: [] + cons: + type: Cons + optional: true + shms: + type: SHM + aggregate: list + default: [] + vms: + type: VM + aggregate: list + default: [] + + VM: + name: + type: string + kernel: + type: string + ram: + type: integer + cpus: + type: integer + cmdline: + type: string + default: "" + initrd: + type: string + optional: true + dtb: + type: string + vbus: + type: string + optional: true + vmnets: + type: string + aggregate: list + default: [] + shms: + type: string + aggregate: list + default: [] + virtio_block: + type: VirtioBlockNode + optional: true + vnets: + type: string + aggregate: list + default: [] + + + VirtioBlockNode: + servers: + type: string + aggregate: list + default: [] + + clients: + type: string + aggregate: list + default: [] + + SHM: + name: + type: string + size: + type: integer + address: + type: integer + optional: true + + Cons: + default_vm: + type: string + optional: true + + VBus: + name: + type: string + devices: + type: Device + aggregate: list + + Device: + name: + type: string + compatible: + type: string + optional: true + mmios: + type: MMIO + aggregate: list + optional: true + irqs: + type: IRQ + aggregate: list + optional: true + MMIO: + address: + type: integer + size: + type: integer + cached: + type: boolean + default: false + IRQ: + irq: + type: integer + type: + type: enum + enum_values: + - SGI + - PPI + - SPI + default: SPI + trigger: + type: enum + enum_values: + - level_high + - rising_edge + +root: HVConfig + +templates: + - io.cfg.j2 + - system.lua diff --git a/ebcl/tools/hypervisor/data/system.lua b/ebcl/tools/hypervisor/data/system.lua new file mode 100755 index 0000000..a6f4d67 --- /dev/null +++ b/ebcl/tools/hypervisor/data/system.lua @@ -0,0 +1,154 @@ +local L4 = require "L4"; +local l = L4.default_loader; + +local max_ds = 16 + +function sched(prio_up, prio_base, cpus) + return L4.Env.user_factory:create(L4.Proto.Scheduler, prio_up, prio_base, cpus); +end + +function create_ds(size_in_bytes, align, lower, upper, flags) + args = { L4.Proto.Dataspace, size_in_bytes, flags, align and align or 0 }; + + if lower then + table.insert(args, string.format("physmin=%x", lower)); + end + + if upper then + table.insert(args, string.format("physmax=%x", upper)); + end + + return L4.Env.user_factory:create(table.unpack(args)):m("rws"); +end + +function start_cons(opts) + l.log_fab = l:new_channel() + + return l:start( + { + log = L4.Env.log, + scheduler = sched(0x44, 0x40, 0x1), + caps = {cons = l.log_fab:svr()}, + }, + "rom/cons " .. opts + ) +end + +function start_io(io_busses, files) + local io_caps = { + sigma0 = L4.cast(L4.Proto.Factory, L4.Env.sigma0):create(L4.Proto.Sigma0), + icu = L4.Env.icu + } + + for k, v in pairs(io_busses) do + local c = l:new_channel() + io_busses[k] = c + io_caps[k] = c:svr() + end + + return l:start( + { + log = { "io", "white" }, + scheduler = sched(0x65, 0x60, 0x1), + caps = io_caps, + }, + "rom/io " .. files + ) +end + +function start_l4vio_net_p2p() + local vnp2p_ipc_gate = l:new_channel() + l:start( + { + caps = {svr = vnp2p_ipc_gate:svr()}, + scheduler = sched(0x44, 0x40, 0x1), + log = {"vnp2p", "yellow"}, + }, + "rom/l4vio_net_p2p" + ) + + return { + portA = L4.cast(L4.Proto.Factory, vnp2p_ipc_gate):create(0, "ds-max=" .. max_ds):m("rwd"), + portB = L4.cast(L4.Proto.Factory, vnp2p_ipc_gate):create(0, "ds-max=" .. max_ds):m("rwd") + } +end + + +function start_vm( + name, + key, + kernel, + rammb, + initrd, + dtb, + cmdline, + cpus, + extra_caps +) + local c = {} + c[#c + 1] = "-i" + c[#c + 1] = "-v" + c[#c + 1] = "-krom/" .. kernel + c[#c + 1] = "-drom/" .. dtb + if initrd then + c[#c + 1] = "-rrom/" .. initrd + end + if cmdline and #cmdline > 0 then + c[#c + 1] = "-c" .. cmdline + end + --c[#c + 1] = "-v" + + local caps = { + ram = create_ds( + rammb * 1024 * 1024, + 21, + nil, + nil, + L4.Mem_alloc_flags.Continuous | L4.Mem_alloc_flags.Pinned | L4.Mem_alloc_flags.Super_pages + ) + } + if extra_caps then + for k, v in pairs(extra_caps) do + caps[k] = v + end + end + + return l:startv( + { + log = {name, "", "key=" .. key}, + scheduler = sched(0x6, 0x1, cpus), + caps = caps + }, + "rom/uvmm", + table.unpack(c) + ) +end + +function start_virtio_block(channel) + local c = {} + local caps = {} + local servers = {} + local clients = {} + for key, value in pairs(channel) do + channel[key] = { + server = l:new_channel(), + client = l:new_channel() + } + local server_name = key .. "_server" + local client_name = key .. "_client" + c[#c + 1] = client_name .. ",filed,ds-max=" .. max_ds .. ",servercap=" .. server_name + caps[server_name] = channel[key].server:svr() + caps[client_name] = channel[key].client:svr() + end + + l:startv( + { + log = { "vioblk", "b" }, + scheduler = sched(0x45, 0x40, 0x1), + caps = caps + }, + "rom/virtio-block", table.unpack(c) + ) +end + +return _ENV diff --git a/ebcl/tools/hypervisor/model.py b/ebcl/tools/hypervisor/model.py new file mode 100644 index 0000000..17ba231 --- /dev/null +++ b/ebcl/tools/hypervisor/model.py @@ -0,0 +1,373 @@ +from __future__ import annotations + +import logging + +from .model_gen import BaseModel + + +class VNet: + """ + Represents a virtual network interface pair + """ + + name: str + """Name of the net to identify it in the device tree and the configuration""" + users: list[VM] + """Users of the vm net""" + + def __init__(self, name: str) -> None: + self.name = name + self.users = [] + + def add_user(self, vm: VM) -> None: + """ + add a user to the virtual interface pair + Since this is a pair, only two users are allowed + """ + if len(self.users) == 2: + logging.error("VMNet %s is already used by two vms", self.name) + self.users.append(vm) + + def __repr__(self) -> str: + return f"VNet({self.name})" + + +class VNetRef: + """ + Reference to one side of a virtual network interface pair + """ + + vnet: VNet + """Link to the vmnet""" + site_a: bool + """True for one of the two sides""" + + def __init__(self, vnet: VNet, site_a: bool) -> None: + self.vnet = vnet + self.site_a = site_a + + def __getattr__(self, attr: str): + return getattr(self.vnet, attr) + + def __repr__(self) -> str: + return f"VNetRef({self.name})" + + +class VirtioBlock: + """ + A virtio block device that has a server and a client interface. + The server interface is a console interfaces that has to be served by vio_filed + The client interface is a standard virtio block interface. + """ + + name: str + """Name used for identification in the configuration and device tree""" + server: VM | None = None + """Server VM""" + client: VM | None = None + """Client VM""" + + def __init__(self, name) -> None: + self.name = name + + def __repr__(self) -> str: + return f"VirtioBlock({self.name})" + + +class VirtioBlockRef: + """ + A reference to the client or server side of a virtio block interface + """ + vio: VirtioBlock + """Link to the virtio block description""" + is_server: bool + """True if this is the link of the server side""" + + def __init__(self, vio: VirtioBlock, is_server: bool) -> None: + self.vio = vio + self.is_server = is_server + + def __getattr__(self, attr: str): + return getattr(self.vio, attr) + + def __repr__(self) -> str: + return f"VirtioBlockRef({self.name})" + + +class MMIO(BaseModel): + """ + A region of memory, that is used by a Device + """ + + address: int + """Start of the region""" + size: int + """Size of the region""" + cached: bool + """If cached is true, the memory is mapped as normal memory instead of device memory""" + +# def __init__(self, config: dict) -> None: +# self.address = config["address"] +# self.size = config["size"] +# self.cached = config["cached"] + + def __repr__(self) -> str: + return f"MMIO(0x{self.address:x}, 0x{self.size:x})" + + +class IRQ(BaseModel): + """ + Describes an interrupt used by a Device. + """ + + irq: int + """The interrupt number""" + is_edge: bool + """If true, the interrupt is rising edge triggered, otherwise it is level high triggered.""" + type: str + """SGI, PPI or SPI""" + trigger: str + """Trigger type enum""" + + def __init__(self, config: dict) -> None: + super().__init__(config) + self.is_edge = self.trigger == "rising_edge" + + @property + def offset(self) -> int: + if self.type == "SGI": + return 0 + elif self.type == "PPI": + return 16 + elif self.type == "SPI": + return 32 + return 0 + + +class Device(BaseModel): + """ + Describes a devices, that is passed through to a vm. + """ + + name: str + """Name of the device""" + compatible: str | None + """Compatible string for mapping the device to the device tree""" + mmios: list[MMIO] + """List of mapped memory regions for the device""" + irqs: list[IRQ] + """List of interrupts used by the device""" + + def __repr__(self) -> str: + return f"Device({self.name})" + + +class VBus(BaseModel): + """ + A virtual bus, that contains devices + """ + + name: str + """Name of the virtual bus used to identify it in the configuration""" + devices: list[Device] + """List of devices that are part of the bus""" + +# def __init__(self, name: str, config: dict) -> None: +# self.name = name +# +# self.devices = [] +# for devname, devconfig in config.items(): +# self.devices.append(Device(devname, devconfig)) + + def __repr__(self) -> str: + return f"VBus({self.name}, {', '.join(map(repr, self.devices))})" + + +class SHM(BaseModel): + """ + A shread memory region, that can be assigned to multiple vms + """ + + name: str + """Name of the region as identifier in the configuration and the device tree""" + size: int + """Size of the region in bytes""" + address: int | None + """Fixed start address of the region, if required""" + + def __lt__(self, other: SHM) -> bool: + """ + Ensure that shared memory segments with a specified address + are allocated first in ascending order + """ + return (self.address and True or False) and (not other.address or self.address < other.address) + + def __repr__(self) -> str: + return f"SHM({self.name})" + + +class Cons(BaseModel): + """Configuration for the console multiplexer""" + + default_vm: str | None + """Automatically attach this vm to cons on startup""" + + +class VirtioBlockNode(BaseModel): + """A Virtio block property of VM""" + servers: list[str] + """List of server virtio block interface names""" + clients: list[str] + """List of client virtio block interface names""" + + +class VM(BaseModel): + """ + Describes a virtual machine + """ + + name: str + """Name of the virtual machine used in console output""" + kernel: str + """The kernel image name""" + ram: int + """The amount of ram assigned to this vm in bytes""" + initrd: str | None + """The initrd file name""" + dtb: str + """The device tree file name""" + cmdline: str + """The kernel command line""" + cpus: int + """The cpu mask to use for this vm (e.g. 0x3 -> CPU 0 and 1)""" + vbus: str | VBus | None + """The virtual bus to connect to this vm""" + vnets: list[str] | list[VNetRef] + """List of virual network pairs used by this vm""" + shms: list[str] | list[SHM] + """List of shared memory regions available to this vm""" + virtio_block = list[VirtioBlockNode] | list[VirtioBlockRef] + + def finalize(self, registry: HVConfig) -> None: + registry.register_module(self.kernel) + registry.register_module(self.dtb) + if self.initrd: + registry.register_module(self.initrd) + + if not isinstance(self.vbus, VBus): + self.vbus = registry.get_vbus(self.vbus) + self.shms = registry.get_shms(self.shms) # type: ignore + self.vnets = list( + map(lambda x: registry.register_vnet(x, self), self.vnets) # type: ignore + ) + + if self.virtio_block: + server = map( + lambda x: registry.register_virtio_block(x, self, True), self.virtio_block.servers # type: ignore + ) + clients = map( + lambda x: registry.register_virtio_block(x, self, False), self.virtio_block.clients # type: ignore + ) + self.virtio_block: list[VirtioBlockRef] = list(server) + list(clients) # type: ignore + else: + self.virtio_block = [] + + def __repr__(self) -> str: + return self.name + + +class HVConfig(BaseModel): + """ + Complete configuration of the hypervisor. + """ + vms: list[VM] + """List of vms defined""" + cons: Cons + """Console mupltiplexer configuration""" + vnets: list[VNet] + """List of virtual network interface pairs""" + vio_block: list[VirtioBlock] + """List of virtio block interface pairs""" + vbus: list[VBus] + """List of registered virtual busses""" + shms: list[SHM] + """List of registered shared memory regions""" + modules: set[str] + """List of required hypervisor modules""" + + def register_vnet(self, name: str, user: VM) -> VNetRef: + """ + Register a virtual network link user. + The vnet is matched by the name and if it does not exist yet it will be created. + """ + for net in self.vnets: + if net.name == name: + vnet = net + break + else: + vnet = VNet(name) + self.vnets.append(vnet) + vnet.add_user(user) + return VNetRef(vnet, len(vnet.users) == 1) + + def register_virtio_block(self, name: str, user: VM, is_server: bool) -> VirtioBlockRef: + """ + Register a virtio block interface user. + The interface is matched by name and if it does not exist yet it will be created. + """ + for cur in self.vio_block: + if cur.name == name: + vio = cur + break + else: + vio = VirtioBlock(name) + self.vio_block.append(vio) + if is_server: + if vio.server: + logging.error("Server for Virtio Block %s already set", vio.name) + else: + vio.server = user + else: + if vio.client: + logging.error("Client for Virtio Block %s already set", vio.name) + else: + vio.client = user + return VirtioBlockRef(vio, is_server) + + def get_vbus(self, name: str | None) -> VBus | None: + """ + Returns an existing virtual bus with the given name + """ + if not name: + return None + for vbus in self.vbus: + if vbus.name == name: + return vbus + logging.error("Vbus %s not defined", name) + return None + + def get_shms(self, names: list[str]) -> list[SHM]: + """ + Returns all registered shared memories matched by names + """ + out = list(filter(lambda x: x.name in names, self.shms)) + if len(out) != len(names): + missing = set(names) - set([out.name for out in out]) + logging.error("Not all used shms are defined: %s", ", ".join(missing)) + return out + + def register_module(self, name: str) -> None: + """ + Add a module to the list of required hypervisor modules. + """ + self.modules.add(name) + + def __init__(self, config: dict) -> None: + self.vnets = [] + self.vio_block = [] + self.modules = set() + + super().__init__(config) + + for vm in self.vms: + vm.finalize(self) diff --git a/ebcl/tools/hypervisor/model_gen.py b/ebcl/tools/hypervisor/model_gen.py new file mode 100644 index 0000000..5cde11d --- /dev/null +++ b/ebcl/tools/hypervisor/model_gen.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import builtins +import logging +from typing import Any, Type + + +class ConfigError(Exception): + """Configuration error""" + + +class PropertyInfo: + """Information about a property of the configuration""" + + name: str + type: str + aggregate: str + default: Any + optional: bool + enum_values: list[str] | None + + def __init__(self, name: str, info: dict) -> None: + self.name = name + self.type = info["type"] + self.aggregate = info.get("aggregate", "None") + self.optional = info.get("optional", False) + self.default = info.get("default", None) + self.enum_values = info.get("enum_values", None) + + def validate_enum(self, value: Any) -> bool: + """ + Validate value of enum. + Note: This is always true, it the type is not an enum + """ + if self.type != "enum": + return True + if not isinstance(value, str) or not self.enum_values: + return False + return value in self.enum_values + + def get_type(self, registry: dict[str, builtins.type[BaseModel]]) -> Type | None: + """Returns the expected class type of a value""" + if self.type == "string" or self.type == "enum": + return str + elif self.type == "integer": + return int + elif self.type == "boolean": + return bool + return registry.get(self.type, None) + + +class BaseModel: + """ + Base class for all model classes + """ + class_registry: dict[str, type[BaseModel]] = {} + PROPERTIES: list[PropertyInfo] + + def __init__(self, config: dict) -> None: + self.__load(config) + + def __parse_type(self, info: PropertyInfo, value: Any) -> Any: + """Verify that value matches the PropertyInfo""" + expected = info.get_type(self.class_registry) + + if not expected: + raise ConfigError(f"Unexpected type for {type(self).__name__}.{info.name}: {info.type}") + + if not info.validate_enum(value): + raise ConfigError( + f"Invalid value for enum type {type(self).__name__}.{info.name}, " + f"expected one of {', '.join(info.enum_values or [])} but is '{value}" + ) + + if issubclass(expected, BaseModel): + value = expected(value) + + if not isinstance(value, expected): + raise ConfigError( + f"Wrong type for {type(self).__name__}.{info.name}, expected {info.type} but is {type(value)}" + ) + return value + + def __load_list(self, info: PropertyInfo, value: Any) -> None: + """Load a list of values""" + if not isinstance(value, list): + logging.warning( + "Value for %s.%s is expected to be a list. It will be converted to a single item list", + type(self).__name__, + info.name + ) + value = [value] + setattr(self, info.name, list(map(lambda x: self.__parse_type(info, x), value))) + + def __load(self, config: dict) -> None: + """Load this instance from the config""" + used_keys = [] + for info in self.PROPERTIES: + value = config.get(info.name, info.default) + if value is None: + if not info.optional: + raise ConfigError(f"Property {info.name} for {type(self).__name__} is not optional") + setattr(self, info.name, None) + continue + used_keys.append(info.name) + + if info.aggregate == "list": + self.__load_list(info, value) + else: + setattr(self, info.name, self.__parse_type(info, value)) + + unused_keys = set(config.keys()) - set(used_keys) + if unused_keys: + logging.warning("Some properties for %s are unused: %s", type(self).__name__, ", ".join(unused_keys)) diff --git a/ebcl/tools/hypervisor/schema_loader.py b/ebcl/tools/hypervisor/schema_loader.py new file mode 100644 index 0000000..871b6cf --- /dev/null +++ b/ebcl/tools/hypervisor/schema_loader.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import importlib.util +import importlib.resources +import logging +from pathlib import Path +import sys +import types +from typing import IO, Literal, Protocol +import yaml + +from .model_gen import BaseModel, ConfigError, PropertyInfo +from . import model +from . import data + + +def merge_dict(old: dict, new: dict) -> None: + """ + Recursively merge one dictionary into another. + For existing values the behavior depends on the datatype: + * strings, numbers and booleans are overwritten + * lists are appended + * dicts are updated by recursively executing this function + """ + def _merge_list(old: list, new: list) -> None: + old += new + + for key, value in new.items(): + if key not in old: + old[key] = value + else: + if type(old[key]) is not type(value): + raise ConfigError(f"Type for {key} do not match ({type(old[key])} != {type(value)})") + + if isinstance(value, list): + _merge_list(old[key], value) + elif isinstance(value, dict): + merge_dict(old[key], value) + elif isinstance(value, (str, int, bool)): + old[key] = value + else: + raise ConfigError(f"Unknown type for {key} ({type(value)})") + + +class DisablePycache: + """ + Temporarily disables creation of the byte cache. + Use this as a context manager, e.g.:: + + with DisablePycache(): + ... + """ + _old_state: bool = False + + def __enter__(self): + self._old_state = sys.dont_write_bytecode + sys.dont_write_bytecode = True + + def __exit__(self, type, value, traceback): + sys.dont_write_bytecode = self._old_state + + +class FileReadProtocol(Protocol): + """ + Protocol for read-only text file interfaces (Path and importlib.Traversable) + """ + def open(self, mode: Literal["r"] = "r", *, encoding: str | None = None, errors: str | None = None) -> IO[str]: + ... + + @property + def name(self) -> str: + ... + + def read_text(self, encoding: str | None = None) -> str: + ... + + +class Schema: + """ + Load base and extension schema + + The class loads schema.yaml from this module and from the extension path. + It then unifies the configuration and creates/updates all model classes + with the properties defined in the schema. + The model is loaded from model.py of this module and also model.py from + the extension path. + This allows fully extending the configuration system with hypervisor + extensions required for specific hypervisor versions. + """ + _root: type[BaseModel] + _templates: list[FileReadProtocol] + _schema_version: int + + def __init__(self, extension: Path | None) -> None: + schema = self._load_base_schema() + ext_model = None + if extension: + ext_schema = self._load_ext_schema(extension) + if ext_schema: + merge_dict(schema, ext_schema) + logging.info("Extension schema loaded") + ext_model = self._load_ext_model(extension) + if ext_model: + logging.info("Extension model loaded") + + for key, value in schema.get("classes", {}).items(): + cls: type[BaseModel] | None = None + if ext_model: + cls = getattr(ext_model, key, None) + if not cls: + cls = getattr(model, key, None) + + if cls and not issubclass(cls, BaseModel): + raise ConfigError(f"Class {cls.__name__} is not derived from BaseModel") + + if not cls: + cls = type(key, (BaseModel,), {}) + cls.PROPERTIES = [PropertyInfo(key, info) for key, info in value.items()] + BaseModel.class_registry[key] = cls + + root = schema.get("root") + if not root or not isinstance(root, str) or root not in BaseModel.class_registry: + raise ConfigError("Missing or invalid root property in schema") + self._root = BaseModel.class_registry[root] + self._templates = self._load_templates(schema.get("templates", []), extension) + + def _load_base_schema(self) -> dict: + """Load the schema.yaml from this module""" + schema_file = importlib.resources.files(data) / "schema.yaml" + with schema_file.open(encoding="utf8") as f: + schema = yaml.load(f, yaml.Loader) + schema_version = schema.get("version", None) + + if not schema_version or not isinstance(schema_version, int): + raise ConfigError("Version missing in base schema") + self._schema_version = schema_version + return schema + + def _load_ext_schema(self, extension: Path) -> dict | None: + """Load the schema.yaml from the extension path""" + schema_file = extension / "schema.yaml" + if not schema_file.is_file(): + return None + + with schema_file.open(encoding="utf-8") as f: + schema = yaml.load(f, yaml.Loader) + schema_ext_version = schema.get("version", None) + if not schema_ext_version or not isinstance(schema_ext_version, int): + raise ConfigError("Version missing in extension schema") + if self._schema_version != schema_ext_version: + raise ConfigError( + f"Version of extension schema ({schema_ext_version}) " + f"does not match base schema version ({self._schema_version})" + ) + return schema + + def _load_ext_model(self, extension: Path) -> None | types.ModuleType: + """Load the model.py from the extension path""" + ext_model_file = extension / "model.py" + if not ext_model_file.exists(): + return None + + spec = importlib.util.spec_from_file_location("ext_model", ext_model_file) + if not spec or not spec.loader: + raise ConfigError(f"Unable to load extension model {ext_model_file}") + ext_model = importlib.util.module_from_spec(spec) + with DisablePycache(): # Disable creation of __pycache__ in extension dir + spec.loader.exec_module(ext_model) + return ext_model + + def _load_templates(self, templates: list[str], extension: Path | None) -> list[FileReadProtocol]: + """Find paths to all templates defined in the schema""" + res: list[FileReadProtocol] = [] + for template in templates: + if extension: + ext_template = extension / template + else: + ext_template = None + if ext_template and ext_template.is_file(): + res.append(ext_template) + else: + path = importlib.resources.files(data) / template + if not path.is_file(): + raise ConfigError(f"Unable to find template {template}") + res.append(path) + + return res + + def parse_config(self, config: dict) -> BaseModel: + """Parse a hypervisor config with the current schema""" + return self._root(config) + + @property + def templates(self) -> list[FileReadProtocol]: + """The templates defined in the schema""" + return self._templates diff --git a/pyproject.toml b/pyproject.toml index 8af387d..5bcd027 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ initrd_generator = "ebcl.tools.initrd.initrd:main" root_generator = "ebcl.tools.root.root:main" root_configurator = "ebcl.tools.root.root_config:main" package_downloader = "ebcl.tools.downloader.downloader:main" +hypervisor_config = "ebcl.tools.hypervisor.config_gen:main" [tool.pylint.format]