diff --git a/ebcl/tools/hypervisor/__init__.py b/ebcl/tools/hypervisor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ebcl/tools/hypervisor/config_gen.py b/ebcl/tools/hypervisor/config_gen.py new file mode 100644 index 0000000..f74bd81 --- /dev/null +++ b/ebcl/tools/hypervisor/config_gen.py @@ -0,0 +1,115 @@ +import argparse +import logging +import os +from pathlib import Path + +import jinja2 +import yaml + +from ebcl.common import init_logging, log_exception +from ebcl.common.files import resolve_file + +from .schema_loader import BaseModel, FileReadProtocol, Schema, merge_dict + + +class BaseResolver: + """ + Resolve bases defined in the yaml config. + """ + + def load(self, config_file: str, conf_dir: str) -> dict: + """ + Load config_file and all of its bases. + """ + config = { + "base": [config_file] + } + while config["base"]: + base_name = config["base"].pop(0) + base_path = resolve_file( + file=base_name, relative_base_dir=conf_dir) + old = config + config = self._load_file(base_path) + merge_dict(config, old) + return config + + def _load_file(self, filename: str) -> dict: + with open(filename, "r", encoding="utf-8") as f: + return yaml.load(f, yaml.Loader) + + +class HvFileGenerator: + """ + Hypervisor configuration file generator + """ + config: BaseModel + schema: Schema + output_path: Path + + def __init__(self, file: str, output_path: str, specialization: str | None = None) -> None: + """ Parse the yaml config file. + Args: + config_file (Path): Path to the yaml config file. + """ + self.output_path = Path(output_path) + + """ Load yaml configuration. """ + config_file = Path(file) + + config = BaseResolver().load(config_file.name, str(config_file.parent)) + del config["base"] + + self.schema = Schema(specialization and Path(specialization) or None) + self.config = self.schema.parse_config(config) + + def _render_template(self, outpath: Path, template: FileReadProtocol) -> None: + """Render a template to target""" + template_obj = jinja2.Template(template.read_text("utf-8"), trim_blocks=True) + + with outpath.open("w", encoding="utf-8") as f: + f.write(template_obj.render(config=self.config)) + + def create_files(self) -> None: + """ + Create three Files of HV + init.ned, io.cfg and modules.list + """ + self.output_path.mkdir(exist_ok=True) + + for template in self.schema.templates: + base, ext = os.path.splitext(template.name) + if ext == ".j2": + logging.info("Rendering %s", base) + outpath = self.output_path / base + self._render_template(outpath, template) + else: + logging.info("Creating %s", template) + outpath = self.output_path / template.name + outpath.write_text(template.read_text("utf-8")) + + +@log_exception(call_exit=True) +def main() -> None: + """ Main entrypoint of EBcL hypervisor generator. """ + init_logging() + + logging.info('\n===================\n' + 'EBcL Hypervisor Config File Generator\n' + '===================\n') + + parser = argparse.ArgumentParser( + description='Create the config files for the hypervisor') + parser.add_argument('-s', '--specialization', type=str, + help='Path to hypervisor specialization directory') + parser.add_argument('config_file', type=str, + help='Path to the YAML configuration file') + parser.add_argument('output', type=str, + help='Path to the output directory') + args = parser.parse_args() + + generator = HvFileGenerator(args.config_file, args.output, args.specialization) + generator.create_files() + + +if __name__ == "__main__": + main() diff --git a/ebcl/tools/hypervisor/data/io.cfg.j2 b/ebcl/tools/hypervisor/data/io.cfg.j2 new file mode 100755 index 0000000..47f0ad7 --- /dev/null +++ b/ebcl/tools/hypervisor/data/io.cfg.j2 @@ -0,0 +1,32 @@ +local Res = Io.Res +local Hw = Io.Hw + +local hw = Io.system_bus(); + +Io.Dt.add_children(hw, function() +{% for vbus in config.vbus %} +{% for dev in vbus.devices %} + {{dev.name}} = Hw.Device(function() +{% if dev.compatible %} + compatible = {"{{dev.compatible}}"} +{% endif %} +{%for mmio in dev.mmios %} + Resource.reg{{loop.index0}} = Res.mmio({{"0x%x"|format(mmio.address)}}, {{"0x%x"|format(mmio.address + mmio.size - 1)}}{% if mmio.cached %}, Io.Resource.F_cached_mem | Io.Resource.F_prefetchable{% endif %}) +{% endfor %} +{%for irq in dev.irqs %} + Resource.irq{{loop.index0}} = Res.irq({{irq.irq + irq.offset}}, Io.Resource.Irq_type_{{irq.is_edge and "raising_edge" or "level_high"}}) +{% endfor %} + end) +{% endfor %} +{% endfor %} +end) + +Io.add_vbusses({ +{% for vbus in config.vbus %} + {{vbus.name}} = Io.Vi.System_bus(function() +{% for dev in vbus.devices %} + {{dev.name}} = wrap(hw.{{dev.name}}) +{% endfor %} + end), +{% endfor %} +}) diff --git a/ebcl/tools/hypervisor/data/model.schema.json b/ebcl/tools/hypervisor/data/model.schema.json new file mode 100644 index 0000000..79e4956 --- /dev/null +++ b/ebcl/tools/hypervisor/data/model.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://linux.elektrobit.com.com/model.schema.json", + "title": "Hypervisor Configuration Model", + "description": "Hypervisor configuration model", + "type": "object", + "additionalProperties": false, + "default": {}, + "properties": { + "version": { + "type": "integer" + }, + "classes": { + "type": "object", + "default": {}, + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": {"type": "string"}, + "default": {"type": ["string", "number", "boolean", "array"]}, + "aggregate": { + "enum": ["None", "list"], + "default": "None" + }, + "optional": {"type": "boolean", "default": false}, + "enum_values": {"type": "array", "items": {"type": "string"}} + } + } + } + }, + "root": { + "type": "string", + "optional": false + }, + "templates": { + "type": "array", + "items": { + "type": "string" + } + } + } +} diff --git a/ebcl/tools/hypervisor/data/modules.list.j2 b/ebcl/tools/hypervisor/data/modules.list.j2 new file mode 100755 index 0000000..e6d2778 --- /dev/null +++ b/ebcl/tools/hypervisor/data/modules.list.j2 @@ -0,0 +1,26 @@ +default-kernel fiasco -serial_esc + +entry hv-image +roottask moe rom/init.ned + +module l4re +module ned +module cons +module io +{% if config.vms %} +module uvmm +{% endif %} +{% if config.vio_block %} +module virtio-block +{% endif %} +{% if config.vnets %} +module l4vio_net_p2p +{% endif %} + +module init.ned +module io.cfg +module system.lua + +{% for module in config.modules %} +module {{module}} +{% endfor %} diff --git a/ebcl/tools/hypervisor/data/schema.yaml b/ebcl/tools/hypervisor/data/schema.yaml new file mode 100644 index 0000000..15d1f16 --- /dev/null +++ b/ebcl/tools/hypervisor/data/schema.yaml @@ -0,0 +1,134 @@ +# yaml-language-server: $schema=model.schema.json + +version: 1 + +classes: + HVConfig: + vbus: + type: VBus + aggregate: list + default: [] + cons: + type: Cons + optional: true + shms: + type: SHM + aggregate: list + default: [] + vms: + type: VM + aggregate: list + default: [] + + VM: + name: + type: string + kernel: + type: string + ram: + type: integer + cpus: + type: integer + cmdline: + type: string + default: "" + initrd: + type: string + optional: true + dtb: + type: string + vbus: + type: string + optional: true + vmnets: + type: string + aggregate: list + default: [] + shms: + type: string + aggregate: list + default: [] + virtio_block: + type: VirtioBlockNode + optional: true + vnets: + type: string + aggregate: list + default: [] + + + VirtioBlockNode: + servers: + type: string + aggregate: list + default: [] + + clients: + type: string + aggregate: list + default: [] + + SHM: + name: + type: string + size: + type: integer + address: + type: integer + optional: true + + Cons: + default_vm: + type: string + optional: true + + VBus: + name: + type: string + devices: + type: Device + aggregate: list + + Device: + name: + type: string + compatible: + type: string + optional: true + mmios: + type: MMIO + aggregate: list + optional: true + irqs: + type: IRQ + aggregate: list + optional: true + MMIO: + address: + type: integer + size: + type: integer + cached: + type: boolean + default: false + IRQ: + irq: + type: integer + type: + type: enum + enum_values: + - SGI + - PPI + - SPI + default: SPI + trigger: + type: enum + enum_values: + - level_high + - rising_edge + +root: HVConfig + +templates: + - io.cfg.j2 + - system.lua diff --git a/ebcl/tools/hypervisor/data/system.lua b/ebcl/tools/hypervisor/data/system.lua new file mode 100755 index 0000000..a6f4d67 --- /dev/null +++ b/ebcl/tools/hypervisor/data/system.lua @@ -0,0 +1,154 @@ +local L4 = require "L4"; +local l = L4.default_loader; + +local max_ds = 16 + +function sched(prio_up, prio_base, cpus) + return L4.Env.user_factory:create(L4.Proto.Scheduler, prio_up, prio_base, cpus); +end + +function create_ds(size_in_bytes, align, lower, upper, flags) + args = { L4.Proto.Dataspace, size_in_bytes, flags, align and align or 0 }; + + if lower then + table.insert(args, string.format("physmin=%x", lower)); + end + + if upper then + table.insert(args, string.format("physmax=%x", upper)); + end + + return L4.Env.user_factory:create(table.unpack(args)):m("rws"); +end + +function start_cons(opts) + l.log_fab = l:new_channel() + + return l:start( + { + log = L4.Env.log, + scheduler = sched(0x44, 0x40, 0x1), + caps = {cons = l.log_fab:svr()}, + }, + "rom/cons " .. opts + ) +end + +function start_io(io_busses, files) + local io_caps = { + sigma0 = L4.cast(L4.Proto.Factory, L4.Env.sigma0):create(L4.Proto.Sigma0), + icu = L4.Env.icu + } + + for k, v in pairs(io_busses) do + local c = l:new_channel() + io_busses[k] = c + io_caps[k] = c:svr() + end + + return l:start( + { + log = { "io", "white" }, + scheduler = sched(0x65, 0x60, 0x1), + caps = io_caps, + }, + "rom/io " .. files + ) +end + +function start_l4vio_net_p2p() + local vnp2p_ipc_gate = l:new_channel() + l:start( + { + caps = {svr = vnp2p_ipc_gate:svr()}, + scheduler = sched(0x44, 0x40, 0x1), + log = {"vnp2p", "yellow"}, + }, + "rom/l4vio_net_p2p" + ) + + return { + portA = L4.cast(L4.Proto.Factory, vnp2p_ipc_gate):create(0, "ds-max=" .. max_ds):m("rwd"), + portB = L4.cast(L4.Proto.Factory, vnp2p_ipc_gate):create(0, "ds-max=" .. max_ds):m("rwd") + } +end + + +function start_vm( + name, + key, + kernel, + rammb, + initrd, + dtb, + cmdline, + cpus, + extra_caps +) + local c = {} + c[#c + 1] = "-i" + c[#c + 1] = "-v" + c[#c + 1] = "-krom/" .. kernel + c[#c + 1] = "-drom/" .. dtb + if initrd then + c[#c + 1] = "-rrom/" .. initrd + end + if cmdline and #cmdline > 0 then + c[#c + 1] = "-c" .. cmdline + end + --c[#c + 1] = "-v" + + local caps = { + ram = create_ds( + rammb * 1024 * 1024, + 21, + nil, + nil, + L4.Mem_alloc_flags.Continuous | L4.Mem_alloc_flags.Pinned | L4.Mem_alloc_flags.Super_pages + ) + } + if extra_caps then + for k, v in pairs(extra_caps) do + caps[k] = v + end + end + + return l:startv( + { + log = {name, "", "key=" .. key}, + scheduler = sched(0x6, 0x1, cpus), + caps = caps + }, + "rom/uvmm", + table.unpack(c) + ) +end + +function start_virtio_block(channel) + local c = {} + local caps = {} + local servers = {} + local clients = {} + for key, value in pairs(channel) do + channel[key] = { + server = l:new_channel(), + client = l:new_channel() + } + local server_name = key .. "_server" + local client_name = key .. "_client" + c[#c + 1] = client_name .. ",filed,ds-max=" .. max_ds .. ",servercap=" .. server_name + caps[server_name] = channel[key].server:svr() + caps[client_name] = channel[key].client:svr() + end + + l:startv( + { + log = { "vioblk", "b" }, + scheduler = sched(0x45, 0x40, 0x1), + caps = caps + }, + "rom/virtio-block", table.unpack(c) + ) +end + +return _ENV diff --git a/ebcl/tools/hypervisor/model.py b/ebcl/tools/hypervisor/model.py new file mode 100644 index 0000000..17ba231 --- /dev/null +++ b/ebcl/tools/hypervisor/model.py @@ -0,0 +1,373 @@ +from __future__ import annotations + +import logging + +from .model_gen import BaseModel + + +class VNet: + """ + Represents a virtual network interface pair + """ + + name: str + """Name of the net to identify it in the device tree and the configuration""" + users: list[VM] + """Users of the vm net""" + + def __init__(self, name: str) -> None: + self.name = name + self.users = [] + + def add_user(self, vm: VM) -> None: + """ + add a user to the virtual interface pair + Since this is a pair, only two users are allowed + """ + if len(self.users) == 2: + logging.error("VMNet %s is already used by two vms", self.name) + self.users.append(vm) + + def __repr__(self) -> str: + return f"VNet({self.name})" + + +class VNetRef: + """ + Reference to one side of a virtual network interface pair + """ + + vnet: VNet + """Link to the vmnet""" + site_a: bool + """True for one of the two sides""" + + def __init__(self, vnet: VNet, site_a: bool) -> None: + self.vnet = vnet + self.site_a = site_a + + def __getattr__(self, attr: str): + return getattr(self.vnet, attr) + + def __repr__(self) -> str: + return f"VNetRef({self.name})" + + +class VirtioBlock: + """ + A virtio block device that has a server and a client interface. + The server interface is a console interfaces that has to be served by vio_filed + The client interface is a standard virtio block interface. + """ + + name: str + """Name used for identification in the configuration and device tree""" + server: VM | None = None + """Server VM""" + client: VM | None = None + """Client VM""" + + def __init__(self, name) -> None: + self.name = name + + def __repr__(self) -> str: + return f"VirtioBlock({self.name})" + + +class VirtioBlockRef: + """ + A reference to the client or server side of a virtio block interface + """ + vio: VirtioBlock + """Link to the virtio block description""" + is_server: bool + """True if this is the link of the server side""" + + def __init__(self, vio: VirtioBlock, is_server: bool) -> None: + self.vio = vio + self.is_server = is_server + + def __getattr__(self, attr: str): + return getattr(self.vio, attr) + + def __repr__(self) -> str: + return f"VirtioBlockRef({self.name})" + + +class MMIO(BaseModel): + """ + A region of memory, that is used by a Device + """ + + address: int + """Start of the region""" + size: int + """Size of the region""" + cached: bool + """If cached is true, the memory is mapped as normal memory instead of device memory""" + +# def __init__(self, config: dict) -> None: +# self.address = config["address"] +# self.size = config["size"] +# self.cached = config["cached"] + + def __repr__(self) -> str: + return f"MMIO(0x{self.address:x}, 0x{self.size:x})" + + +class IRQ(BaseModel): + """ + Describes an interrupt used by a Device. + """ + + irq: int + """The interrupt number""" + is_edge: bool + """If true, the interrupt is rising edge triggered, otherwise it is level high triggered.""" + type: str + """SGI, PPI or SPI""" + trigger: str + """Trigger type enum""" + + def __init__(self, config: dict) -> None: + super().__init__(config) + self.is_edge = self.trigger == "rising_edge" + + @property + def offset(self) -> int: + if self.type == "SGI": + return 0 + elif self.type == "PPI": + return 16 + elif self.type == "SPI": + return 32 + return 0 + + +class Device(BaseModel): + """ + Describes a devices, that is passed through to a vm. + """ + + name: str + """Name of the device""" + compatible: str | None + """Compatible string for mapping the device to the device tree""" + mmios: list[MMIO] + """List of mapped memory regions for the device""" + irqs: list[IRQ] + """List of interrupts used by the device""" + + def __repr__(self) -> str: + return f"Device({self.name})" + + +class VBus(BaseModel): + """ + A virtual bus, that contains devices + """ + + name: str + """Name of the virtual bus used to identify it in the configuration""" + devices: list[Device] + """List of devices that are part of the bus""" + +# def __init__(self, name: str, config: dict) -> None: +# self.name = name +# +# self.devices = [] +# for devname, devconfig in config.items(): +# self.devices.append(Device(devname, devconfig)) + + def __repr__(self) -> str: + return f"VBus({self.name}, {', '.join(map(repr, self.devices))})" + + +class SHM(BaseModel): + """ + A shread memory region, that can be assigned to multiple vms + """ + + name: str + """Name of the region as identifier in the configuration and the device tree""" + size: int + """Size of the region in bytes""" + address: int | None + """Fixed start address of the region, if required""" + + def __lt__(self, other: SHM) -> bool: + """ + Ensure that shared memory segments with a specified address + are allocated first in ascending order + """ + return (self.address and True or False) and (not other.address or self.address < other.address) + + def __repr__(self) -> str: + return f"SHM({self.name})" + + +class Cons(BaseModel): + """Configuration for the console multiplexer""" + + default_vm: str | None + """Automatically attach this vm to cons on startup""" + + +class VirtioBlockNode(BaseModel): + """A Virtio block property of VM""" + servers: list[str] + """List of server virtio block interface names""" + clients: list[str] + """List of client virtio block interface names""" + + +class VM(BaseModel): + """ + Describes a virtual machine + """ + + name: str + """Name of the virtual machine used in console output""" + kernel: str + """The kernel image name""" + ram: int + """The amount of ram assigned to this vm in bytes""" + initrd: str | None + """The initrd file name""" + dtb: str + """The device tree file name""" + cmdline: str + """The kernel command line""" + cpus: int + """The cpu mask to use for this vm (e.g. 0x3 -> CPU 0 and 1)""" + vbus: str | VBus | None + """The virtual bus to connect to this vm""" + vnets: list[str] | list[VNetRef] + """List of virual network pairs used by this vm""" + shms: list[str] | list[SHM] + """List of shared memory regions available to this vm""" + virtio_block = list[VirtioBlockNode] | list[VirtioBlockRef] + + def finalize(self, registry: HVConfig) -> None: + registry.register_module(self.kernel) + registry.register_module(self.dtb) + if self.initrd: + registry.register_module(self.initrd) + + if not isinstance(self.vbus, VBus): + self.vbus = registry.get_vbus(self.vbus) + self.shms = registry.get_shms(self.shms) # type: ignore + self.vnets = list( + map(lambda x: registry.register_vnet(x, self), self.vnets) # type: ignore + ) + + if self.virtio_block: + server = map( + lambda x: registry.register_virtio_block(x, self, True), self.virtio_block.servers # type: ignore + ) + clients = map( + lambda x: registry.register_virtio_block(x, self, False), self.virtio_block.clients # type: ignore + ) + self.virtio_block: list[VirtioBlockRef] = list(server) + list(clients) # type: ignore + else: + self.virtio_block = [] + + def __repr__(self) -> str: + return self.name + + +class HVConfig(BaseModel): + """ + Complete configuration of the hypervisor. + """ + vms: list[VM] + """List of vms defined""" + cons: Cons + """Console mupltiplexer configuration""" + vnets: list[VNet] + """List of virtual network interface pairs""" + vio_block: list[VirtioBlock] + """List of virtio block interface pairs""" + vbus: list[VBus] + """List of registered virtual busses""" + shms: list[SHM] + """List of registered shared memory regions""" + modules: set[str] + """List of required hypervisor modules""" + + def register_vnet(self, name: str, user: VM) -> VNetRef: + """ + Register a virtual network link user. + The vnet is matched by the name and if it does not exist yet it will be created. + """ + for net in self.vnets: + if net.name == name: + vnet = net + break + else: + vnet = VNet(name) + self.vnets.append(vnet) + vnet.add_user(user) + return VNetRef(vnet, len(vnet.users) == 1) + + def register_virtio_block(self, name: str, user: VM, is_server: bool) -> VirtioBlockRef: + """ + Register a virtio block interface user. + The interface is matched by name and if it does not exist yet it will be created. + """ + for cur in self.vio_block: + if cur.name == name: + vio = cur + break + else: + vio = VirtioBlock(name) + self.vio_block.append(vio) + if is_server: + if vio.server: + logging.error("Server for Virtio Block %s already set", vio.name) + else: + vio.server = user + else: + if vio.client: + logging.error("Client for Virtio Block %s already set", vio.name) + else: + vio.client = user + return VirtioBlockRef(vio, is_server) + + def get_vbus(self, name: str | None) -> VBus | None: + """ + Returns an existing virtual bus with the given name + """ + if not name: + return None + for vbus in self.vbus: + if vbus.name == name: + return vbus + logging.error("Vbus %s not defined", name) + return None + + def get_shms(self, names: list[str]) -> list[SHM]: + """ + Returns all registered shared memories matched by names + """ + out = list(filter(lambda x: x.name in names, self.shms)) + if len(out) != len(names): + missing = set(names) - set([out.name for out in out]) + logging.error("Not all used shms are defined: %s", ", ".join(missing)) + return out + + def register_module(self, name: str) -> None: + """ + Add a module to the list of required hypervisor modules. + """ + self.modules.add(name) + + def __init__(self, config: dict) -> None: + self.vnets = [] + self.vio_block = [] + self.modules = set() + + super().__init__(config) + + for vm in self.vms: + vm.finalize(self) diff --git a/ebcl/tools/hypervisor/model_gen.py b/ebcl/tools/hypervisor/model_gen.py new file mode 100644 index 0000000..5cde11d --- /dev/null +++ b/ebcl/tools/hypervisor/model_gen.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import builtins +import logging +from typing import Any, Type + + +class ConfigError(Exception): + """Configuration error""" + + +class PropertyInfo: + """Information about a property of the configuration""" + + name: str + type: str + aggregate: str + default: Any + optional: bool + enum_values: list[str] | None + + def __init__(self, name: str, info: dict) -> None: + self.name = name + self.type = info["type"] + self.aggregate = info.get("aggregate", "None") + self.optional = info.get("optional", False) + self.default = info.get("default", None) + self.enum_values = info.get("enum_values", None) + + def validate_enum(self, value: Any) -> bool: + """ + Validate value of enum. + Note: This is always true, it the type is not an enum + """ + if self.type != "enum": + return True + if not isinstance(value, str) or not self.enum_values: + return False + return value in self.enum_values + + def get_type(self, registry: dict[str, builtins.type[BaseModel]]) -> Type | None: + """Returns the expected class type of a value""" + if self.type == "string" or self.type == "enum": + return str + elif self.type == "integer": + return int + elif self.type == "boolean": + return bool + return registry.get(self.type, None) + + +class BaseModel: + """ + Base class for all model classes + """ + class_registry: dict[str, type[BaseModel]] = {} + PROPERTIES: list[PropertyInfo] + + def __init__(self, config: dict) -> None: + self.__load(config) + + def __parse_type(self, info: PropertyInfo, value: Any) -> Any: + """Verify that value matches the PropertyInfo""" + expected = info.get_type(self.class_registry) + + if not expected: + raise ConfigError(f"Unexpected type for {type(self).__name__}.{info.name}: {info.type}") + + if not info.validate_enum(value): + raise ConfigError( + f"Invalid value for enum type {type(self).__name__}.{info.name}, " + f"expected one of {', '.join(info.enum_values or [])} but is '{value}" + ) + + if issubclass(expected, BaseModel): + value = expected(value) + + if not isinstance(value, expected): + raise ConfigError( + f"Wrong type for {type(self).__name__}.{info.name}, expected {info.type} but is {type(value)}" + ) + return value + + def __load_list(self, info: PropertyInfo, value: Any) -> None: + """Load a list of values""" + if not isinstance(value, list): + logging.warning( + "Value for %s.%s is expected to be a list. It will be converted to a single item list", + type(self).__name__, + info.name + ) + value = [value] + setattr(self, info.name, list(map(lambda x: self.__parse_type(info, x), value))) + + def __load(self, config: dict) -> None: + """Load this instance from the config""" + used_keys = [] + for info in self.PROPERTIES: + value = config.get(info.name, info.default) + if value is None: + if not info.optional: + raise ConfigError(f"Property {info.name} for {type(self).__name__} is not optional") + setattr(self, info.name, None) + continue + used_keys.append(info.name) + + if info.aggregate == "list": + self.__load_list(info, value) + else: + setattr(self, info.name, self.__parse_type(info, value)) + + unused_keys = set(config.keys()) - set(used_keys) + if unused_keys: + logging.warning("Some properties for %s are unused: %s", type(self).__name__, ", ".join(unused_keys)) diff --git a/ebcl/tools/hypervisor/schema_loader.py b/ebcl/tools/hypervisor/schema_loader.py new file mode 100644 index 0000000..871b6cf --- /dev/null +++ b/ebcl/tools/hypervisor/schema_loader.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import importlib.util +import importlib.resources +import logging +from pathlib import Path +import sys +import types +from typing import IO, Literal, Protocol +import yaml + +from .model_gen import BaseModel, ConfigError, PropertyInfo +from . import model +from . import data + + +def merge_dict(old: dict, new: dict) -> None: + """ + Recursively merge one dictionary into another. + For existing values the behavior depends on the datatype: + * strings, numbers and booleans are overwritten + * lists are appended + * dicts are updated by recursively executing this function + """ + def _merge_list(old: list, new: list) -> None: + old += new + + for key, value in new.items(): + if key not in old: + old[key] = value + else: + if type(old[key]) is not type(value): + raise ConfigError(f"Type for {key} do not match ({type(old[key])} != {type(value)})") + + if isinstance(value, list): + _merge_list(old[key], value) + elif isinstance(value, dict): + merge_dict(old[key], value) + elif isinstance(value, (str, int, bool)): + old[key] = value + else: + raise ConfigError(f"Unknown type for {key} ({type(value)})") + + +class DisablePycache: + """ + Temporarily disables creation of the byte cache. + Use this as a context manager, e.g.:: + + with DisablePycache(): + ... + """ + _old_state: bool = False + + def __enter__(self): + self._old_state = sys.dont_write_bytecode + sys.dont_write_bytecode = True + + def __exit__(self, type, value, traceback): + sys.dont_write_bytecode = self._old_state + + +class FileReadProtocol(Protocol): + """ + Protocol for read-only text file interfaces (Path and importlib.Traversable) + """ + def open(self, mode: Literal["r"] = "r", *, encoding: str | None = None, errors: str | None = None) -> IO[str]: + ... + + @property + def name(self) -> str: + ... + + def read_text(self, encoding: str | None = None) -> str: + ... + + +class Schema: + """ + Load base and extension schema + + The class loads schema.yaml from this module and from the extension path. + It then unifies the configuration and creates/updates all model classes + with the properties defined in the schema. + The model is loaded from model.py of this module and also model.py from + the extension path. + This allows fully extending the configuration system with hypervisor + extensions required for specific hypervisor versions. + """ + _root: type[BaseModel] + _templates: list[FileReadProtocol] + _schema_version: int + + def __init__(self, extension: Path | None) -> None: + schema = self._load_base_schema() + ext_model = None + if extension: + ext_schema = self._load_ext_schema(extension) + if ext_schema: + merge_dict(schema, ext_schema) + logging.info("Extension schema loaded") + ext_model = self._load_ext_model(extension) + if ext_model: + logging.info("Extension model loaded") + + for key, value in schema.get("classes", {}).items(): + cls: type[BaseModel] | None = None + if ext_model: + cls = getattr(ext_model, key, None) + if not cls: + cls = getattr(model, key, None) + + if cls and not issubclass(cls, BaseModel): + raise ConfigError(f"Class {cls.__name__} is not derived from BaseModel") + + if not cls: + cls = type(key, (BaseModel,), {}) + cls.PROPERTIES = [PropertyInfo(key, info) for key, info in value.items()] + BaseModel.class_registry[key] = cls + + root = schema.get("root") + if not root or not isinstance(root, str) or root not in BaseModel.class_registry: + raise ConfigError("Missing or invalid root property in schema") + self._root = BaseModel.class_registry[root] + self._templates = self._load_templates(schema.get("templates", []), extension) + + def _load_base_schema(self) -> dict: + """Load the schema.yaml from this module""" + schema_file = importlib.resources.files(data) / "schema.yaml" + with schema_file.open(encoding="utf8") as f: + schema = yaml.load(f, yaml.Loader) + schema_version = schema.get("version", None) + + if not schema_version or not isinstance(schema_version, int): + raise ConfigError("Version missing in base schema") + self._schema_version = schema_version + return schema + + def _load_ext_schema(self, extension: Path) -> dict | None: + """Load the schema.yaml from the extension path""" + schema_file = extension / "schema.yaml" + if not schema_file.is_file(): + return None + + with schema_file.open(encoding="utf-8") as f: + schema = yaml.load(f, yaml.Loader) + schema_ext_version = schema.get("version", None) + if not schema_ext_version or not isinstance(schema_ext_version, int): + raise ConfigError("Version missing in extension schema") + if self._schema_version != schema_ext_version: + raise ConfigError( + f"Version of extension schema ({schema_ext_version}) " + f"does not match base schema version ({self._schema_version})" + ) + return schema + + def _load_ext_model(self, extension: Path) -> None | types.ModuleType: + """Load the model.py from the extension path""" + ext_model_file = extension / "model.py" + if not ext_model_file.exists(): + return None + + spec = importlib.util.spec_from_file_location("ext_model", ext_model_file) + if not spec or not spec.loader: + raise ConfigError(f"Unable to load extension model {ext_model_file}") + ext_model = importlib.util.module_from_spec(spec) + with DisablePycache(): # Disable creation of __pycache__ in extension dir + spec.loader.exec_module(ext_model) + return ext_model + + def _load_templates(self, templates: list[str], extension: Path | None) -> list[FileReadProtocol]: + """Find paths to all templates defined in the schema""" + res: list[FileReadProtocol] = [] + for template in templates: + if extension: + ext_template = extension / template + else: + ext_template = None + if ext_template and ext_template.is_file(): + res.append(ext_template) + else: + path = importlib.resources.files(data) / template + if not path.is_file(): + raise ConfigError(f"Unable to find template {template}") + res.append(path) + + return res + + def parse_config(self, config: dict) -> BaseModel: + """Parse a hypervisor config with the current schema""" + return self._root(config) + + @property + def templates(self) -> list[FileReadProtocol]: + """The templates defined in the schema""" + return self._templates diff --git a/pyproject.toml b/pyproject.toml index f425ef5..698eaa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ initrd_generator = "ebcl.tools.initrd.initrd:main" root_generator = "ebcl.tools.root.root:main" root_configurator = "ebcl.tools.root.root_config:main" package_downloader = "ebcl.tools.downloader.downloader:main" +hypervisor_config = "ebcl.tools.hypervisor.config_gen:main" [tool.pylint.format]