From ce098f6426744510f1e6f634a05245096919a8e4 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Thu, 9 Jan 2020 13:10:29 +0100 Subject: [PATCH 1/9] Add map access feature for lists --- yangson/instance.py | 27 +++++++++++++++++++++++++++ yangson/instvalue.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/yangson/instance.py b/yangson/instance.py index e705ac5..283bc65 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -137,6 +137,9 @@ def __init__(self, key: InstanceKey, value: Value, self.value = value # type: Value """Value of the receiver.""" + """" Mapping from key tuple to children """ + self._childmap = None # type: dict + @property def name(self) -> InstanceName: """Name of the receiver.""" @@ -178,6 +181,8 @@ def __getitem__(self, key: InstanceKey) -> "InstanceNode": `name`. InstanceValueError: If the receiver's value is not an object. """ + if isinstance(self.value, ArrayValue) and isinstance(key, tuple): + return self._mapentry(key) if isinstance(self.value, ObjectValue): return self._member(key) if isinstance(self.value, ArrayValue): @@ -397,6 +402,28 @@ def _entry(self, index: int) -> "ArrayEntry": except (IndexError, TypeError): raise NonexistentInstance(self.json_pointer(), "entry " + str(index)) from None + def _mapentry(self, key: tuple) -> "ArrayEntry": + if self._childmap is None: + self._childmap = {} + keys = self.schema_node._key_members + + # iterate over all childs + for child in self: + keylist = [] + + # collect key values into tuple + for keyit in keys: + keylist.append(child[keyit].value) + + # cache mapping + self._childmap[tuple(keylist)] = child + + try: + return self._childmap[key] + except (KeyError): + raise NonexistentInstance(self.json_pointer(), + f"key '{key}'") from None + def _peek_schema_route(self, sroute: SchemaRoute) -> Value: irt = InstanceRoute() sn = self.schema_node diff --git a/yangson/instvalue.py b/yangson/instvalue.py index 515964f..9e7c39c 100644 --- a/yangson/instvalue.py +++ b/yangson/instvalue.py @@ -35,7 +35,7 @@ EntryValue = Union[ScalarValue, "ObjectValue"] """Type of the value a list ot leaf-list entry.""" -InstanceKey = Union[InstanceName, int] +InstanceKey = Union[InstanceName, int, tuple] """Index of an array entry or name of an object member.""" MetadataObject = Dict[PrefName, ScalarValue] From 2a9a9c309be7bcee921eedd199300cb3951098b1 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Mon, 20 Jan 2020 10:41:19 +0100 Subject: [PATCH 2/9] Speedup tuple based lookup and add map based lookup of childs --- yangson/instance.py | 68 ++++++++++++++++++++++++++++++++------------ yangson/instvalue.py | 2 +- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/yangson/instance.py b/yangson/instance.py index 283bc65..eeb9b33 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -137,8 +137,10 @@ def __init__(self, key: InstanceKey, value: Value, self.value = value # type: Value """Value of the receiver.""" - """" Mapping from key tuple to children """ - self._childmap = None # type: dict + """Mapping from key tuple to children""" + self._childmap = {} # type: dict + """Remember at which index we want to start parsing the childs""" + self._parse_next = 0 # type: int @property def name(self) -> InstanceName: @@ -183,12 +185,19 @@ def __getitem__(self, key: InstanceKey) -> "InstanceNode": """ if isinstance(self.value, ArrayValue) and isinstance(key, tuple): return self._mapentry(key) + if isinstance(self.value, ArrayValue) and isinstance(key, dict): + return self._mapentry(self._map2tuple(key)) if isinstance(self.value, ObjectValue): return self._member(key) if isinstance(self.value, ArrayValue): return self._entry(key) raise InstanceValueError(self.json_pointer(), "scalar instance") + def __contains__(self, key: InstanceKey) -> bool: + """Checks if key does exist + """ + return self.get(key) is not None + def __iter__(self): """Return receiver's iterator. @@ -210,6 +219,14 @@ def ita(): return iter(self._member_names()) raise InstanceValueError(self.json_pointer(), "scalar instance") + def get(self, key: InstanceKey, d=None): + """Return member or entry with given key, returns default if it does not exist + """ + try: + return self[key] + except (InstanceValueError, NonexistentInstance): + return d + def is_internal(self) -> bool: """Return ``True`` if the receiver is an instance of an internal node. """ @@ -402,27 +419,42 @@ def _entry(self, index: int) -> "ArrayEntry": except (IndexError, TypeError): raise NonexistentInstance(self.json_pointer(), "entry " + str(index)) from None + def _map2tuple(self, key: dict) -> tuple: + """generate tuple for key""" + keylist = [] + for keyit in self.schema_node._key_members: + keylist.append(key[keyit]) + + return tuple(keylist) + def _mapentry(self, key: tuple) -> "ArrayEntry": - if self._childmap is None: - self._childmap = {} - keys = self.schema_node._key_members + child = self._childmap.get(key) + if child is not None: + return child - # iterate over all childs - for child in self: - keylist = [] + """lazy initialization of mapping from keys to childnodes""" + keys = self.schema_node._key_members - # collect key values into tuple - for keyit in keys: - keylist.append(child[keyit].value) + """iterate over all childs starting with last unparsed index""" + while self._parse_next < len(self.value): + child = self[self._parse_next] - # cache mapping - self._childmap[tuple(keylist)] = child + """generate tuple for key""" + keylist = [] + for keyit in keys: + keylist.append(child[keyit].value) + keytuple = tuple(keylist) - try: - return self._childmap[key] - except (KeyError): - raise NonexistentInstance(self.json_pointer(), - f"key '{key}'") from None + """cache mapping for later use""" + self._childmap[keytuple] = child + + """mark this child as done""" + self._parse_next = self._parse_next + 1; + + if keytuple == key: + return child + + raise NonexistentInstance(self.json_pointer(), f"key '{key}'") from None def _peek_schema_route(self, sroute: SchemaRoute) -> Value: irt = InstanceRoute() diff --git a/yangson/instvalue.py b/yangson/instvalue.py index 9e7c39c..cdb1bda 100644 --- a/yangson/instvalue.py +++ b/yangson/instvalue.py @@ -35,7 +35,7 @@ EntryValue = Union[ScalarValue, "ObjectValue"] """Type of the value a list ot leaf-list entry.""" -InstanceKey = Union[InstanceName, int, tuple] +InstanceKey = Union[InstanceName, int, tuple, dict] """Index of an array entry or name of an object member.""" MetadataObject = Dict[PrefName, ScalarValue] From 00ddffad40564f8f3b382367123fa4ef84f778e9 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Tue, 21 Apr 2020 09:30:33 +0200 Subject: [PATCH 3/9] Remove caching for member access for now --- yangson/instance.py | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/yangson/instance.py b/yangson/instance.py index eeb9b33..de5d98f 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -137,11 +137,6 @@ def __init__(self, key: InstanceKey, value: Value, self.value = value # type: Value """Value of the receiver.""" - """Mapping from key tuple to children""" - self._childmap = {} # type: dict - """Remember at which index we want to start parsing the childs""" - self._parse_next = 0 # type: int - @property def name(self) -> InstanceName: """Name of the receiver.""" @@ -428,30 +423,11 @@ def _map2tuple(self, key: dict) -> tuple: return tuple(keylist) def _mapentry(self, key: tuple) -> "ArrayEntry": - child = self._childmap.get(key) - if child is not None: - return child - - """lazy initialization of mapping from keys to childnodes""" - keys = self.schema_node._key_members - - """iterate over all childs starting with last unparsed index""" - while self._parse_next < len(self.value): - child = self[self._parse_next] - + """iterate over all childs""" + for child in self: """generate tuple for key""" - keylist = [] - for keyit in keys: - keylist.append(child[keyit].value) - keytuple = tuple(keylist) - - """cache mapping for later use""" - self._childmap[keytuple] = child - - """mark this child as done""" - self._parse_next = self._parse_next + 1; - - if keytuple == key: + childkey = tuple(child[singlekey].value for singlekey in self.schema_node._key_members) + if key == childkey: return child raise NonexistentInstance(self.json_pointer(), f"key '{key}'") from None From c01c561b150a50432e66e4a8cfe8eb677f66ac53 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Thu, 30 Apr 2020 08:59:52 +0200 Subject: [PATCH 4/9] Add xml support --- yangson/datamodel.py | 19 +++++++-- yangson/datatype.py | 22 +++++++++++ yangson/instance.py | 48 ++++++++++++++++++++-- yangson/schemadata.py | 15 ++++++- yangson/schemanode.py | 92 ++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 187 insertions(+), 9 deletions(-) diff --git a/yangson/datamodel.py b/yangson/datamodel.py index c3ba82e..bcf9d49 100644 --- a/yangson/datamodel.py +++ b/yangson/datamodel.py @@ -25,6 +25,7 @@ import hashlib import json from typing import Optional, Tuple +import xml.etree.ElementTree as ET from .enumerations import ContentType from .exceptions import BadYangLibraryData from .instance import (InstanceRoute, InstanceIdParser, ResourceIdParser, @@ -75,13 +76,13 @@ def __init__(self, yltxt: str, mod_path: Tuple[str] = (".",), ModuleNotFound: If a YANG module wasn't found in any of the directories specified in `mod_path`. """ - self.schema = SchemaTreeNode() - self.schema._ctype = ContentType.all try: self.yang_library = json.loads(yltxt) except json.JSONDecodeError as e: raise BadYangLibraryData(str(e)) from None self.schema_data = SchemaData(self.yang_library, mod_path) + self.schema = SchemaTreeNode(self.schema_data) + self.schema._ctype = ContentType.all self._build_schema() self.schema.description = description if description else ( "Data model ID: " + @@ -107,7 +108,19 @@ def from_raw(self, robj: RawObject) -> RootNode: Root instance node. """ cooked = self.schema.from_raw(robj) - return RootNode(cooked, self.schema, cooked.timestamp) + return RootNode(cooked, self.schema, self.schema_data, cooked.timestamp) + + def from_xml(self, root: ET.Element) -> RootNode: + """Create an instance node from a raw data tree. + + Args: + robj: Dictionary representing a raw data tree. + + Returns: + Root instance node. + """ + cooked = self.schema.from_xml(root) + return RootNode(cooked, self.schema, self.schema_data, cooked.timestamp) def get_schema_node(self, path: SchemaPath) -> Optional[SchemaNode]: """Return the schema node addressed by a schema path. diff --git a/yangson/datatype.py b/yangson/datatype.py index 0b94bab..a137b2f 100644 --- a/yangson/datatype.py +++ b/yangson/datatype.py @@ -97,10 +97,25 @@ def from_raw(self, raw: RawScalar) -> Optional[ScalarValue]: if isinstance(raw, str): return raw + def from_xml(self, xml: str) -> Optional[ScalarValue]: + """Return a cooked value of the received XML type. + + Args: + xml: Text of the XML node + """ + return self.from_raw(xml) + def to_raw(self, val: ScalarValue) -> Optional[RawScalar]: """Return a raw value ready to be serialized in JSON.""" return val + def to_xml(self, val: ScalarValue) -> Optional[str]: + """Return XML text value ready to be serialized in XML.""" + value = self.to_raw(val) + if value is not None: + return str(value) + return None + def parse_value(self, text: str) -> Optional[ScalarValue]: """Parse value of the receiver's type. @@ -229,6 +244,13 @@ def from_raw(self, raw: RawScalar) -> Optional[Tuple[None]]: if raw == [None]: return (None,) + def from_xml(self, xml: str) -> Optional[Tuple[None]]: + if xml == '': + return (None,) + + def to_xml(self, val: Tuple[None]) -> None: + return None + class BitsType(DataType): """Class representing YANG "bits" type.""" diff --git a/yangson/instance.py b/yangson/instance.py index e705ac5..c46adc2 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -33,6 +33,7 @@ import json from typing import Dict, List, Optional, Tuple, Union from urllib.parse import unquote +import xml.etree.ElementTree as ET from .enumerations import ContentType, ValidationScope from .exceptions import (BadSchemaNodeType, EndOfInput, InstanceException, InstanceValueError, InvalidKeyValue, @@ -157,6 +158,13 @@ def path(self) -> Tuple[InstanceKey]: inst = inst.parinst return tuple(res) + @property + def schema_data(self): + inst: InstanceNode = self + while inst.parinst: + inst = inst.parinst + return inst._schema_data + def __str__(self) -> str: """Return string representation of the receiver's value.""" sn = self.schema_node @@ -372,6 +380,40 @@ def raw_value(self) -> RawValue: return [en.raw_value() for en in self] return self.schema_node.type.to_raw(self.value) + def to_xml(self, element: ET.Element = None, schema_data: "SchemaData" = None): + """put receiver's value into a XML element""" + if schema_data is None: + schema_data = self.schema_data + if isinstance(self.value, ObjectValue): + if element is None: + element = ET.Element(self.schema_node.name) + element.attrib['xmlns'] = self.schema_node.ns + for cname in self: + if cname[:1] == '@': + # ignore annotations for now until they are stored independent of JSON encoding + continue + m = self[cname] + sn = m.schema_node + dp = sn.data_parent() + + if isinstance(m.schema_node, (ListNode, LeafListNode)): + for en in m: + child = ET.SubElement(element, sn.name) + if not dp or dp.ns != sn.ns: + child.attrib['xmlns'] = schema_data.modules_by_name[sn.ns].xml_namespace + en.to_xml(child, schema_data) + else: + child = ET.SubElement(element, sn.name) + if not dp or dp.ns != sn.ns: + child.attrib['xmlns'] = schema_data.modules_by_name[sn.ns].xml_namespace + m.to_xml(child, schema_data) + elif isinstance(self.value, ArrayValue): + # Array outside an Object doesn't make sense + super().to_xml(element) + else: + element.text = self.schema_node.type.to_xml(self.value) + return element + def _member_names(self) -> List[InstanceName]: if isinstance(self.value, ObjectValue): return [m for m in self.value if not m.startswith("@")] @@ -484,8 +526,9 @@ class RootNode(InstanceNode): """This class represents the root of the instance tree.""" def __init__(self, value: Value, schema_node: "DataNode", - timestamp: datetime): + schema_data: "SchemaData", timestamp: datetime): super().__init__("/", value, None, schema_node, timestamp) + self._schema_data = schema_data def up(self) -> None: """Override the superclass method. @@ -497,7 +540,7 @@ def up(self) -> None: def _copy(self, newval: Value, newts: datetime = None) -> InstanceNode: return RootNode( - newval, self.schema_node, newts if newts else newval.timestamp) + newval, self.schema_node, self._schema_data, newts if newts else newval.timestamp) def _ancestors_or_self( self, qname: Union[QualName, bool] = None) -> List["RootNode"]: @@ -509,7 +552,6 @@ def _ancestors( """XPath - return the list of receiver's ancestors.""" return [] - class ObjectMember(InstanceNode): """This class represents an object member.""" diff --git a/yangson/schemadata.py b/yangson/schemadata.py index ec042a9..97683c8 100644 --- a/yangson/schemadata.py +++ b/yangson/schemadata.py @@ -70,10 +70,14 @@ def __init__(self, main_module: YangIdentifier): """Set of supported features.""" self.main_module = main_module # type: ModuleId """Main module of the receiver.""" + self.xml_namespace = None # type: str + """Content of the namespace definition of the module""" self.prefix_map = {} # type: Dict[YangIdentifier, ModuleId] """Map of prefixes to module identifiers.""" self.statement = None # type: Statement """Corresponding (sub)module statements.""" + self.path = None # type: str + """Path to the yang file this module was initialized from""" self.submodules = set() # type: MutableSet[ModuleId] """Set of submodules.""" @@ -96,6 +100,9 @@ def __init__(self, yang_lib: Dict[str, Any], mod_path: List[str]) -> None: """List of directories where to look for YANG modules.""" self.modules = {} # type: Dict[ModuleId, ModuleData] """Dictionary of module data.""" + self.modules_by_name = {} # type: Dict[str, ModuleData] + """Dictionary of module data by module name.""" + self.modules_by_ns = {} self._module_sequence = [] # type: List[ModuleId] """List that defines the order of module processing.""" self._from_yang_library(yang_lib) @@ -121,12 +128,15 @@ def _from_yang_library(self, yang_lib: Dict[str, Any]) -> None: rev = item["revision"] mid = (name, rev) mdata = ModuleData(mid) + mdata.xml_namespace = item.get('namespace') self.modules[mid] = mdata + self.modules_by_name[name] = mdata + self.modules_by_ns[mdata.xml_namespace] = mdata if item["conformance-type"] == "implement": if name in self.implement: raise MultipleImplementedRevisions(name) self.implement[name] = rev - mod = self._load_module(name, rev) + mod = self._load_module(name, rev, mdata) mdata.statement = mod if "feature" in item: mdata.features.update(item["feature"]) @@ -150,7 +160,7 @@ def _from_yang_library(self, yang_lib: Dict[str, Any]) -> None: self._check_feature_dependences() def _load_module(self, name: YangIdentifier, - rev: RevisionDate) -> Statement: + rev: RevisionDate, mdata: ModuleData) -> Statement: """Read and parse a YANG module or submodule.""" for d in self.module_search_path: run = 0 @@ -162,6 +172,7 @@ def _load_module(self, name: YangIdentifier, try: with open(fn, encoding='utf-8') as infile: res = ModuleParser(infile.read(), name, rev).parse() + mdata.path = fn except (FileNotFoundError, PermissionError, ModuleContentMismatch): run += 1 continue diff --git a/yangson/schemanode.py b/yangson/schemanode.py index 4117857..06582d2 100644 --- a/yangson/schemanode.py +++ b/yangson/schemanode.py @@ -43,6 +43,7 @@ from datetime import datetime from typing import Any, Dict, List, MutableSet, Optional, Set, Tuple +import xml.etree.ElementTree as ET from .constraint import Must from .datatype import (DataType, LinkType, RawScalar, IdentityrefType) @@ -158,6 +159,20 @@ def from_raw(self, rval: RawValue, jptr: JSONPointer = "") -> Value: """ raise NotImplementedError + def from_xml(self, rval: ET.Element, jptr: JSONPointer = "") -> Value: + """Return instance value transformed from a raw value using receiver. + + Args: + rval: XML node. + jptr: JSON pointer of the current instance node. + + Raises: + RawMemberError: If a member inside `rval` is not defined in the + schema. + RawTypeError: If a scalar value inside `rval` is of incorrect type. + """ + raise NotImplementedError + def clear_val_counters(self) -> None: """Clear receiver's validation counter.""" self.val_count = 0 @@ -451,6 +466,38 @@ def from_raw(self, rval: RawObject, jptr: JSONPointer = "") -> ObjectValue: res[ch.iname()] = ch.from_raw(rval[qn], npath) return res + def from_xml(self, rval: ET.Element, jptr: JSONPointer = "") -> Value: + res = ObjectValue() + if jptr == "": + self._process_xml_child(res, None, rval, jptr) + else: + for xmlchild in rval: + self._process_xml_child(res, rval, xmlchild, jptr) + return res + + def _process_xml_child(self, res: ObjectValue, + rval: ET.Element, xmlchild: ET.Element, jptr: JSONPointer = ""): + if xmlchild.tag[0] == '{': + xmlns, name = xmlchild.tag[1:].split('}') + ns = self.schema_root().schema_data.modules_by_ns.get(xmlns).main_module[0] + qn = ns + ':' + name + else: + name = qn = xmlchild.tag + ns = self.ns + + ch = self.get_data_child(name, ns) + npath = jptr + "/" + qn + if ch is None: + raise RawMemberError(npath) + + if isinstance(ch, SequenceNode): + if ch.iname() in res: + # already done when we discovered an earlier element of the array + return + res[ch.iname()] = ch.from_xml(rval, qn, npath) + else: + res[ch.iname()] = ch.from_xml(xmlchild, npath) + def _process_metadata(self, rmo: RawMetadataObject, jptr: JSONPointer) -> MetadataObject: res = {} @@ -721,10 +768,11 @@ def _flatten(self) -> List[SchemaNode]: class SchemaTreeNode(GroupNode): """Root node of a schema tree.""" - def __init__(self): + def __init__(self, schemadata: "SchemaData" = None): """Initialize the class instance.""" super().__init__() self.annotations: Dict[QualName, Annotation] = {} + self.schema_data = schemadata def data_parent(self) -> InternalNode: """Override the superclass method.""" @@ -850,6 +898,12 @@ def from_raw(self, rval: RawScalar, jptr: JSONPointer = "") -> ScalarValue: raise RawTypeError(jptr, self.type.yang_type() + " value") return res + def from_xml(self, rval: ET.Element, jptr: JSONPointer = "") -> Value: + res = self.type.from_xml(rval.text) + if res is None: + raise RawTypeError(jptr, self.type.yang_type() + " value") + return res + def _node_digest(self) -> Dict[str, Any]: res = super()._node_digest() res["type"] = self.type._type_digest(self.config) @@ -1016,6 +1070,25 @@ def from_raw(self, rval: RawList, jptr: JSONPointer = "") -> ArrayValue: res.append(self.entry_from_raw(en, f"{jptr}/{i}")) return res + def from_xml(self, rval: ET.Element, tagname: str, jptr: JSONPointer = "") -> Value: + res = ArrayValue() + i = 0 + for xmlchild in rval: + if xmlchild.tag[0] == '{': + xmlns, name = xmlchild.tag[1:].split('}') + ns = self.schema_root().schema_data.modules_by_ns.get(xmlns).main_module[0] + qn = ns + ':' + name + else: + name = qn = xmlchild.tag + ns = self.ns + if qn != tagname: + # just collect the array + continue + + npath = jptr + "/" + str(i) + res.append(self.entry_from_xml(xmlchild, npath)) + return res + def entry_from_raw(self, rval: RawEntry, jptr: JSONPointer = "") -> EntryValue: """Transform a raw (leaf-)list entry into the cooked form. @@ -1031,6 +1104,20 @@ def entry_from_raw(self, rval: RawEntry, """ return super().from_raw(rval, jptr) + def entry_from_xml(self, rval: ET.Element, jptr: JSONPointer = "") -> EntryValue: + """Transform a XML (leaf-)list entry into the cooked form. + + Args: + rval: xml node + jptr: JSON pointer of the entry + + Raises: + NonexistentSchemaNode: If a member inside `rval` is not defined + in the schema. + RawTypeError: If a scalar value inside `rval` is of incorrect type. + """ + return super().from_xml(rval, jptr) + class ListNode(SequenceNode, InternalNode): """List node.""" @@ -1325,6 +1412,9 @@ def convert(val): return res return convert(rval) + def from_xml(self, rval: ET.Element, jptr: JSONPointer = "") -> Value: + super().from_xml(rval, jptr) + def _default_instance(self, pnode: "InstanceNode", ctype: ContentType, lazy: bool = False) -> "InstanceNode": return pnode From 80e6d0b25af60771e0b1797c5e22e11f3655cdb3 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Thu, 30 Apr 2020 11:40:33 +0200 Subject: [PATCH 5/9] Fixes for module loader --- yangson/schemadata.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/yangson/schemadata.py b/yangson/schemadata.py index 97683c8..ba7cf2a 100644 --- a/yangson/schemadata.py +++ b/yangson/schemadata.py @@ -145,11 +145,15 @@ def _from_yang_library(self, yang_lib: Dict[str, Any]) -> None: if "submodule" in item: for s in item["submodule"]: sname = s["name"] - smid = (sname, s["revision"]) - sdata = ModuleData(mid) + srev = s["revision"] + smid = (sname, srev) + sdata = ModuleData(smid) + sdata.xml_namespace = s.get('namespace') self.modules[smid] = sdata + self.modules_by_name[sname] = sdata + self.modules_by_ns[sdata.xml_namespace] = sdata mdata.submodules.add(smid) - submod = self._load_module(*smid) + submod = self._load_module(sname, srev, sdata) sdata.statement = submod bt = submod.find1("belongs-to", name, required=True) locpref = bt.find1("prefix", required=True).argument From 2573f1557d89fac509428b066d4eae43851a1de1 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Tue, 5 May 2020 08:19:23 +0200 Subject: [PATCH 6/9] Fix several issues with XML API, especially with IdentityRef --- yangson/datatype.py | 25 +++++++++++--- yangson/exceptions.py | 10 ++++++ yangson/instance.py | 1 + yangson/schemanode.py | 66 +++++++++++++++++++++++-------------- yangson/xmlparser.py | 77 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+), 29 deletions(-) create mode 100644 yangson/xmlparser.py diff --git a/yangson/datatype.py b/yangson/datatype.py index a137b2f..afcec92 100644 --- a/yangson/datatype.py +++ b/yangson/datatype.py @@ -47,12 +47,13 @@ import base64 import decimal import numbers +import xml.etree.ElementTree as ET from typing import Any, Dict, List, Optional, Tuple, Union, TYPE_CHECKING from .constraint import Intervals, Pattern from .exceptions import ( InvalidArgument, ParserException, ModuleNotRegistered, UnknownPrefix, - InvalidLeafrefPath) + InvalidLeafrefPath, MissingModuleNamespace) from .schemadata import SchemaContext from .instance import InstanceNode, InstanceIdParser, InstanceRoute from .statement import Statement @@ -97,13 +98,13 @@ def from_raw(self, raw: RawScalar) -> Optional[ScalarValue]: if isinstance(raw, str): return raw - def from_xml(self, xml: str) -> Optional[ScalarValue]: + def from_xml(self, xml: ET.Element) -> Optional[ScalarValue]: """Return a cooked value of the received XML type. Args: xml: Text of the XML node """ - return self.from_raw(xml) + return self.from_raw(xml.text) def to_raw(self, val: ScalarValue) -> Optional[RawScalar]: """Return a raw value ready to be serialized in JSON.""" @@ -604,6 +605,22 @@ def from_raw(self, raw: RawScalar) -> Optional[QualName]: return None return (i2, i1) if s else (i1, self.sctx.default_ns) + def from_xml(self, xml: ET.Element) -> Optional[QualName]: + try: + i1, s, i2 = xml.text.partition(":") + except AttributeError: + return None + if not i1: + return (i2, self.sctx.default_ns) + + ns_url = xml.attrib.get('xmlns:'+i1) + if not ns_url: + raise MissingModuleNamespace(ns_url) + module = self.sctx.schema_data.modules_by_ns.get(ns_url) + if not module: + raise MissingModuleNamespace(ns_url) + return (i2, module.main_module[0]) + def __contains__(self, val: QualName) -> bool: for b in self.bases: if not self.sctx.schema_data.is_derived_from(val, b): @@ -741,7 +758,7 @@ def parse_value(self, text: str) -> Optional[int]: return None def from_raw(self, raw: RawScalar) -> Optional[int]: - if not isinstance(raw, int) or isinstance(raw, bool): + if not isinstance(raw, (int, bool, str)): return None try: return int(raw) diff --git a/yangson/exceptions.py b/yangson/exceptions.py index d0f7f7a..515b11b 100644 --- a/yangson/exceptions.py +++ b/yangson/exceptions.py @@ -231,6 +231,16 @@ def __str__(self) -> str: return self.name +class MissingModuleNamespace(YangsonException): + """Abstract exception class – a module is missing.""" + + def __init__(self, ns: str): + self.ns = ns + + def __str__(self) -> str: + return self.ns + + class ModuleContentMismatch(YangsonException): """Abstract exception class – unexpected module name or revision.""" diff --git a/yangson/instance.py b/yangson/instance.py index c46adc2..f61c062 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -552,6 +552,7 @@ def _ancestors( """XPath - return the list of receiver's ancestors.""" return [] + class ObjectMember(InstanceNode): """This class represents an object member.""" diff --git a/yangson/schemanode.py b/yangson/schemanode.py index 06582d2..045e2ef 100644 --- a/yangson/schemanode.py +++ b/yangson/schemanode.py @@ -50,7 +50,8 @@ from .enumerations import Axis, ContentType, DefaultDeny, ValidationScope from .exceptions import ( AnnotationTypeError, InvalidArgument, - MissingAnnotationTarget, MissingAugmentTarget, RawMemberError, + MissingAnnotationTarget, MissingAugmentTarget, MissingModuleNamespace, + RawMemberError, RawTypeError, SchemaError, SemanticError, UndefinedAnnotation, YangsonException, YangTypeError) from .instvalue import ( @@ -159,7 +160,7 @@ def from_raw(self, rval: RawValue, jptr: JSONPointer = "") -> Value: """ raise NotImplementedError - def from_xml(self, rval: ET.Element, jptr: JSONPointer = "") -> Value: + def from_xml(self, rval: ET.Element, jptr: JSONPointer = "", isroot: bool = False) -> Value: """Return instance value transformed from a raw value using receiver. Args: @@ -466,17 +467,18 @@ def from_raw(self, rval: RawObject, jptr: JSONPointer = "") -> ObjectValue: res[ch.iname()] = ch.from_raw(rval[qn], npath) return res - def from_xml(self, rval: ET.Element, jptr: JSONPointer = "") -> Value: + def from_xml(self, rval: ET.Element, jptr: JSONPointer = "", isroot: bool = False) -> ObjectValue: res = ObjectValue() - if jptr == "": - self._process_xml_child(res, None, rval, jptr) + if isroot: + self._process_xmlobj_child(res, None, rval, jptr) else: for xmlchild in rval: - self._process_xml_child(res, rval, xmlchild, jptr) + self._process_xmlobj_child(res, rval, xmlchild, jptr) return res - def _process_xml_child(self, res: ObjectValue, - rval: ET.Element, xmlchild: ET.Element, jptr: JSONPointer = ""): + def _process_xmlobj_child( + self, res: ObjectValue, rval: ET.Element, + xmlchild: ET.Element, jptr: JSONPointer = ""): if xmlchild.tag[0] == '{': xmlns, name = xmlchild.tag[1:].split('}') ns = self.schema_root().schema_data.modules_by_ns.get(xmlns).main_module[0] @@ -494,7 +496,7 @@ def _process_xml_child(self, res: ObjectValue, if ch.iname() in res: # already done when we discovered an earlier element of the array return - res[ch.iname()] = ch.from_xml(rval, qn, npath) + res[ch.iname()] = ch.from_xml(rval, npath, qn) else: res[ch.iname()] = ch.from_xml(xmlchild, npath) @@ -899,7 +901,7 @@ def from_raw(self, rval: RawScalar, jptr: JSONPointer = "") -> ScalarValue: return res def from_xml(self, rval: ET.Element, jptr: JSONPointer = "") -> Value: - res = self.type.from_xml(rval.text) + res = self.type.from_xml(rval) if res is None: raise RawTypeError(jptr, self.type.yang_type() + " value") return res @@ -1070,25 +1072,39 @@ def from_raw(self, rval: RawList, jptr: JSONPointer = "") -> ArrayValue: res.append(self.entry_from_raw(en, f"{jptr}/{i}")) return res - def from_xml(self, rval: ET.Element, tagname: str, jptr: JSONPointer = "") -> Value: + def from_xml(self, rval: ET.Element, jptr: JSONPointer = "", + tagname: str = None, isroot: bool = False) -> ArrayValue: res = ArrayValue() i = 0 - for xmlchild in rval: - if xmlchild.tag[0] == '{': - xmlns, name = xmlchild.tag[1:].split('}') - ns = self.schema_root().schema_data.modules_by_ns.get(xmlns).main_module[0] - qn = ns + ':' + name - else: - name = qn = xmlchild.tag - ns = self.ns - if qn != tagname: - # just collect the array - continue - - npath = jptr + "/" + str(i) - res.append(self.entry_from_xml(xmlchild, npath)) + if isroot: + return self._process_xmlarray_child(res, rval, jptr) + else: + for xmlchild in rval: + self._process_xmlarray_child( + res, xmlchild, jptr + "/" + str(i)) return res + def _process_xmlarray_child( + self, res: ArrayValue, xmlchild: ET.Element, + jptr: JSONPointer = "", tagname: str = None): + if xmlchild.tag[0] == '{': + xmlns, name = xmlchild.tag[1:].split('}') + module = self.schema_root().schema_data.modules_by_ns.get(xmlns) + if not module: + raise MissingModuleNamespace(xmlns) + ns = module.main_module[0] + qn = ns + ':' + name + else: + name = qn = xmlchild.tag + ns = self.ns + if tagname and qn != tagname: + # just collect the array + return + + child = self.entry_from_xml(xmlchild, jptr) + res.append(child) + return child + def entry_from_raw(self, rval: RawEntry, jptr: JSONPointer = "") -> EntryValue: """Transform a raw (leaf-)list entry into the cooked form. diff --git a/yangson/xmlparser.py b/yangson/xmlparser.py new file mode 100644 index 0000000..ddca82b --- /dev/null +++ b/yangson/xmlparser.py @@ -0,0 +1,77 @@ +# Copyright © 2016-2020 CZ.NIC, z. s. p. o. +# +# This file is part of Yangson. +# +# Yangson is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) any +# later version. +# +# Yangson is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Yangson. If not, see . + +"""Extended XML parser that preserves the xmlns attributes +""" + +import xml.etree.ElementTree as ET + + +class XMLParser(ET.XMLPullParser): + ''' + Extended XML parser that add namespaces to ELements as + xmlns/xmlns:... attributes + ''' + def __init__(self, source: str = None): + ''' + Initialize the XML parser + + Args: + source: optional string containing the whole XML document + ''' + super().__init__(events=['start', 'start-ns', 'end-ns', 'end']) + + self._root = None + self._nslist = list() + self._namespaces = {} + + if source: + self.feed(source) + self.close() + self.parse() + + def feed(self, xml: str): + '''Feed additional data to the XML parser''' + super().feed(xml) + + def close(self): + '''End XML data stream''' + super().close() + + def parse(self): + '''Parse all events in current available data''' + for ev_type, ev_data in super().read_events(): + if ev_type == 'start-ns': + ns_name, ns_url = ev_data + self._namespaces[ns_name] = ns_url + self._nslist.append(ns_name) + elif ev_type == 'end-ns': + ns_name = self._nslist.pop() + del self._namespaces[ns_name] + elif ev_type == 'start' and self._root is None: + self._root = ev_data + elif ev_type == 'end': + for ns_name, ns_url in self._namespaces.items(): + attr = 'xmlns' if ns_name == '' else 'xmlns:'+ns_name + ev_data.attrib[attr] = ns_url + + @property + def root(self): + '''Return root node + + Only valid if the first event has been parsed''' + return self._root From 239b807281f8a82d375cd5213c020da32c539162 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Wed, 6 May 2020 16:09:49 +0200 Subject: [PATCH 7/9] Support filter function for JSON and XML export --- yangson/instance.py | 109 +++++++++++++++++++++++++++++++------------- 1 file changed, 77 insertions(+), 32 deletions(-) diff --git a/yangson/instance.py b/yangson/instance.py index f61c062..957e19e 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -31,12 +31,13 @@ from datetime import datetime import json -from typing import Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from urllib.parse import unquote import xml.etree.ElementTree as ET from .enumerations import ContentType, ValidationScope from .exceptions import (BadSchemaNodeType, EndOfInput, InstanceException, InstanceValueError, InvalidKeyValue, + MissingModuleNamespace, NonexistentInstance, NonDataNode, NonexistentSchemaNode, UnexpectedInput) from .instvalue import (ArrayValue, InstanceKey, ObjectValue, Value, @@ -50,6 +51,14 @@ "InstanceException", "InstanceValueError", "NonexistentInstance"] +class OutputFilter: + def begin_child(self, parent: "InstanceNode", node: "InstanceNode")->bool: + return True + + def end_child(self, parent: "InstanceNode", node: "InstanceNode")->bool: + return True + + class LinkedList: """Persistent linked list of instance values.""" @@ -133,6 +142,8 @@ def __init__(self, key: InstanceKey, value: Value, """Parent instance node, or ``None`` for the root node.""" self.schema_node = schema_node # type: DataNode """Data node corresponding to the instance node.""" + self.schema_data = parinst.schema_data if parinst else None + """Link to schema data""" self.timestamp = timestamp # type: datetime """Time of the receiver's last modification.""" self.value = value # type: Value @@ -158,13 +169,6 @@ def path(self) -> Tuple[InstanceKey]: inst = inst.parinst return tuple(res) - @property - def schema_data(self): - inst: InstanceNode = self - while inst.parinst: - inst = inst.parinst - return inst._schema_data - def __str__(self) -> str: """Return string representation of the receiver's value.""" sn = self.schema_node @@ -372,44 +376,85 @@ def add_defaults(self, ctype: ContentType = None) -> "InstanceNode": break return res.up() - def raw_value(self) -> RawValue: + def raw_value(self, filter: OutputFilter = OutputFilter()) -> RawValue: """Return receiver's value in a raw form (ready for JSON encoding).""" if isinstance(self.value, ObjectValue): - return {m: self._member(m).raw_value() for m in self.value} + value = {} + for m in self.value: + member = self[m] + add1 = filter.begin_child(self, member) + if add1: + member_value = member.raw_value(filter) + add2 = filter.end_child(self, member) + if add1 and add2: + value[m] = member_value + return value if isinstance(self.value, ArrayValue): - return [en.raw_value() for en in self] + value = list() + for en in self: + member_value = en.raw_value(filter) + if member_value: + value.append(member_value) + return value return self.schema_node.type.to_raw(self.value) - def to_xml(self, element: ET.Element = None, schema_data: "SchemaData" = None): + def to_xml(self, filter: OutputFilter = OutputFilter(), elem: ET.Element = None): """put receiver's value into a XML element""" - if schema_data is None: - schema_data = self.schema_data + if elem is None: + element = ET.Element(self.schema_node.name) + + module = self.schema_data.modules_by_name.get(self.schema_node.ns) + if not module: + raise MissingModuleNamespace(self.schema_node.ns) + element.attrib['xmlns'] = module.xml_namespace + else: + element = elem + if isinstance(self.value, ObjectValue): - if element is None: - element = ET.Element(self.schema_node.name) - element.attrib['xmlns'] = self.schema_node.ns for cname in self: + childs = list() if cname[:1] == '@': # ignore annotations for now until they are stored independent of JSON encoding continue - m = self[cname] - sn = m.schema_node - dp = sn.data_parent() - if isinstance(m.schema_node, (ListNode, LeafListNode)): - for en in m: + m = self[cname] + if filter.begin_child(self, m): + sn = m.schema_node + dp = sn.data_parent() + + if isinstance(m.schema_node, (ListNode, LeafListNode)): + for en in m: + add1 = filter.begin_child(self, en) + if add1: + child = ET.SubElement(element, sn.name) + childs.append(child) + if not dp or dp.ns != sn.ns: + module = self.schema_data.modules_by_name.get(sn.ns) + if not module: + raise MissingModuleNamespace(sn.ns) + child.attrib['xmlns'] = module.xml_namespace + en.to_xml(filter, child) + add2 = filter.end_child(self, en) + if add1 and not add2: + element.remove(child) + childs.remove(child) + else: child = ET.SubElement(element, sn.name) + childs.append(child) if not dp or dp.ns != sn.ns: - child.attrib['xmlns'] = schema_data.modules_by_name[sn.ns].xml_namespace - en.to_xml(child, schema_data) - else: - child = ET.SubElement(element, sn.name) - if not dp or dp.ns != sn.ns: - child.attrib['xmlns'] = schema_data.modules_by_name[sn.ns].xml_namespace - m.to_xml(child, schema_data) + module = self.schema_data.modules_by_name.get(sn.ns) + if not module: + raise MissingModuleNamespace(sn.ns) + child.attrib['xmlns'] = module.xml_namespace + m.to_xml(filter, child) + if not filter.end_child(self, m): + for c in childs: + element.remove(c) + if elem is None and len(element) == 0: + return None elif isinstance(self.value, ArrayValue): # Array outside an Object doesn't make sense - super().to_xml(element) + super().to_xml(filter, element) else: element.text = self.schema_node.type.to_xml(self.value) return element @@ -528,7 +573,7 @@ class RootNode(InstanceNode): def __init__(self, value: Value, schema_node: "DataNode", schema_data: "SchemaData", timestamp: datetime): super().__init__("/", value, None, schema_node, timestamp) - self._schema_data = schema_data + self.schema_data = schema_data def up(self) -> None: """Override the superclass method. @@ -540,7 +585,7 @@ def up(self) -> None: def _copy(self, newval: Value, newts: datetime = None) -> InstanceNode: return RootNode( - newval, self.schema_node, self._schema_data, newts if newts else newval.timestamp) + newval, self.schema_node, self.schema_data, newts if newts else newval.timestamp) def _ancestors_or_self( self, qname: Union[QualName, bool] = None) -> List["RootNode"]: From a04536c0b7128201449d20756bdb0fe6fc1a4384 Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Thu, 7 May 2020 08:19:17 +0200 Subject: [PATCH 8/9] Add second filter function for adding array elements Cleanup XML element generation --- yangson/instance.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/yangson/instance.py b/yangson/instance.py index 56da04c..277fd18 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -52,10 +52,16 @@ class OutputFilter: - def begin_child(self, parent: "InstanceNode", node: "InstanceNode")->bool: + def begin_member(self, parent: "InstanceNode", node: "InstanceNode")->bool: return True - def end_child(self, parent: "InstanceNode", node: "InstanceNode")->bool: + def end_member(self, parent: "InstanceNode", node: "InstanceNode")->bool: + return True + + def begin_element(self, parent: "InstanceNode", node: "InstanceNode")->bool: + return True + + def end_element(self, parent: "InstanceNode", node: "InstanceNode")->bool: return True @@ -399,18 +405,21 @@ def raw_value(self, filter: OutputFilter = OutputFilter()) -> RawValue: value = {} for m in self.value: member = self[m] - add1 = filter.begin_child(self, member) + add1 = filter.begin_member(self, member) if add1: member_value = member.raw_value(filter) - add2 = filter.end_child(self, member) + add2 = filter.end_member(self, member) if add1 and add2: value[m] = member_value return value if isinstance(self.value, ArrayValue): value = list() for en in self: - member_value = en.raw_value(filter) - if member_value: + add1 = filter.begin_element(self, en) + if add1: + member_value = en.raw_value(filter) + add2 = filter.end_element(self, en) + if add1 and add2 and member_value: value.append(member_value) return value return self.schema_node.type.to_raw(self.value) @@ -435,28 +444,26 @@ def to_xml(self, filter: OutputFilter = OutputFilter(), elem: ET.Element = None) continue m = self[cname] - if filter.begin_child(self, m): + if filter.begin_member(self, m): sn = m.schema_node dp = sn.data_parent() if isinstance(m.schema_node, (ListNode, LeafListNode)): for en in m: - add1 = filter.begin_child(self, en) + add1 = filter.begin_element(m, en) if add1: - child = ET.SubElement(element, sn.name) - childs.append(child) + child = ET.Element(sn.name) if not dp or dp.ns != sn.ns: module = self.schema_data.modules_by_name.get(sn.ns) if not module: raise MissingModuleNamespace(sn.ns) child.attrib['xmlns'] = module.xml_namespace en.to_xml(filter, child) - add2 = filter.end_child(self, en) - if add1 and not add2: - element.remove(child) - childs.remove(child) + add2 = filter.end_element(m, en) + if add1 and add2: + childs.append(child) else: - child = ET.SubElement(element, sn.name) + child = ET.Element(sn.name) childs.append(child) if not dp or dp.ns != sn.ns: module = self.schema_data.modules_by_name.get(sn.ns) @@ -464,9 +471,9 @@ def to_xml(self, filter: OutputFilter = OutputFilter(), elem: ET.Element = None) raise MissingModuleNamespace(sn.ns) child.attrib['xmlns'] = module.xml_namespace m.to_xml(filter, child) - if not filter.end_child(self, m): + if filter.end_member(self, m): for c in childs: - element.remove(c) + element.append(c) if elem is None and len(element) == 0: return None elif isinstance(self.value, ArrayValue): From d6e1203cfc46356c73c8f724493be44686c9a3fc Mon Sep 17 00:00:00 2001 From: Henning Rogge Date: Thu, 7 May 2020 08:19:17 +0200 Subject: [PATCH 9/9] Add second filter function for adding array elements Cleanup XML element generation --- yangson/instance.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/yangson/instance.py b/yangson/instance.py index 957e19e..360090e 100644 --- a/yangson/instance.py +++ b/yangson/instance.py @@ -52,10 +52,16 @@ class OutputFilter: - def begin_child(self, parent: "InstanceNode", node: "InstanceNode")->bool: + def begin_member(self, parent: "InstanceNode", node: "InstanceNode")->bool: return True - def end_child(self, parent: "InstanceNode", node: "InstanceNode")->bool: + def end_member(self, parent: "InstanceNode", node: "InstanceNode")->bool: + return True + + def begin_element(self, parent: "InstanceNode", node: "InstanceNode")->bool: + return True + + def end_element(self, parent: "InstanceNode", node: "InstanceNode")->bool: return True @@ -382,18 +388,21 @@ def raw_value(self, filter: OutputFilter = OutputFilter()) -> RawValue: value = {} for m in self.value: member = self[m] - add1 = filter.begin_child(self, member) + add1 = filter.begin_member(self, member) if add1: member_value = member.raw_value(filter) - add2 = filter.end_child(self, member) + add2 = filter.end_member(self, member) if add1 and add2: value[m] = member_value return value if isinstance(self.value, ArrayValue): value = list() for en in self: - member_value = en.raw_value(filter) - if member_value: + add1 = filter.begin_element(self, en) + if add1: + member_value = en.raw_value(filter) + add2 = filter.end_element(self, en) + if add1 and add2 and member_value: value.append(member_value) return value return self.schema_node.type.to_raw(self.value) @@ -418,28 +427,26 @@ def to_xml(self, filter: OutputFilter = OutputFilter(), elem: ET.Element = None) continue m = self[cname] - if filter.begin_child(self, m): + if filter.begin_member(self, m): sn = m.schema_node dp = sn.data_parent() if isinstance(m.schema_node, (ListNode, LeafListNode)): for en in m: - add1 = filter.begin_child(self, en) + add1 = filter.begin_element(m, en) if add1: - child = ET.SubElement(element, sn.name) - childs.append(child) + child = ET.Element(sn.name) if not dp or dp.ns != sn.ns: module = self.schema_data.modules_by_name.get(sn.ns) if not module: raise MissingModuleNamespace(sn.ns) child.attrib['xmlns'] = module.xml_namespace en.to_xml(filter, child) - add2 = filter.end_child(self, en) - if add1 and not add2: - element.remove(child) - childs.remove(child) + add2 = filter.end_element(m, en) + if add1 and add2: + childs.append(child) else: - child = ET.SubElement(element, sn.name) + child = ET.Element(sn.name) childs.append(child) if not dp or dp.ns != sn.ns: module = self.schema_data.modules_by_name.get(sn.ns) @@ -447,9 +454,9 @@ def to_xml(self, filter: OutputFilter = OutputFilter(), elem: ET.Element = None) raise MissingModuleNamespace(sn.ns) child.attrib['xmlns'] = module.xml_namespace m.to_xml(filter, child) - if not filter.end_child(self, m): + if filter.end_member(self, m): for c in childs: - element.remove(c) + element.append(c) if elem is None and len(element) == 0: return None elif isinstance(self.value, ArrayValue):