dtlib: implement copy.deepcopy() for DT

The standard library copy module allows you to implement shallow and
deep copies of objects. See its documentation for more details on
these terms.

Implementing copy.deepcopy() support for DT objects will allow us to
"clone" devicetree objects in other classes. This in turn will enable
new features, such as native system devicetree support, within the
python-devicetree.

It is also a pure feature extension which can't harm anything and is
therefore safe to merge now, even if system devicetree is never
adopted in Zephyr.

Note that we are making use of the move from OrderedDict to regular
dict to make this implementation more convenient.

See https://github.com/devicetree-org/lopper/ for more information on
system devicetree. We want to add system devicetree support to dtlib
because it seems to be a useful way to model modern, heterogeneous
SoCs than traditional devicetree, which can really only model a single
CPU "cluster" within such an SoC.

In order to create 'regular' devicetrees from a system devicetree, we
will want a programming interface that does the following:

   1. parse the system devicetree
   2. receive the desired transformations on it
   3. perform the desired transformations to make
      a 'regular' devicetree

Step 3 can be done as a destructive modification on an object-oriented
representation of a system devicetree, and that's the approach we will
take in python-devicetree. It will therefore be convenient to have an
efficient deepcopy implementation to be able to preserve the original
system devicetree and the derived regular devicetree in memory in the
same python process.

Signed-off-by: Martí Bolívar <marti.bolivar@nordicsemi.no>

scripts/dts/python-devicetree/src/devicetree/dtlib.py

@@ -88,6 +88,8 @@ def __init__(self, name: str, parent: Optional['Node'], dt: 'DT'):
         """
         Node constructor. Not meant to be called directly by clients.
         """
+        # Remember to update DT.__deepcopy__() if you change this.
+
         self.name = name
         self.props: Dict[str, 'Property'] = {}
         self.nodes: Dict[str, 'Node'] = {}
@@ -282,6 +284,8 @@ class Property:
     #
 
     def __init__(self, node: Node, name: str):
+        # Remember to update DT.__deepcopy__() if you change this.
+
         if "@" in name:
             node.dt._parse_error("'@' is only allowed in node names")
 
@@ -716,14 +720,15 @@ class DT:
     # Public interface
     #
 
-    def __init__(self, filename: str, include_path: Iterable[str] = (),
+    def __init__(self, filename: Optional[str], include_path: Iterable[str] = (),
                  force: bool = False):
         """
         Parses a DTS file to create a DT instance. Raises OSError if 'filename'
         can't be opened, and DTError for any parse errors.
 
         filename:
-          Path to the .dts file to parse.
+          Path to the .dts file to parse. (If None, an empty devicetree
+          is created; this is unlikely to be what you want.)
 
         include_path:
           An iterable (e.g. list or tuple) containing paths to search for
@@ -734,6 +739,8 @@ def __init__(self, filename: str, include_path: Iterable[str] = (),
           Try not to raise DTError even if the input tree has errors.
           For experimental use; results not guaranteed.
         """
+        # Remember to update __deepcopy__() if you change this.
+
         self._root: Optional[Node] = None
         self.alias2node: Dict[str, Node] = {}
         self.label2node: Dict[str, Node] = {}
@@ -745,7 +752,8 @@ def __init__(self, filename: str, include_path: Iterable[str] = (),
 
         self._force = force
 
-        self._parse_file(filename, include_path)
+        if filename is not None:
+            self._parse_file(filename, include_path)
 
     @property
     def root(self) -> Node:
@@ -844,6 +852,102 @@ def __repr__(self):
                 f"include_path={self._include_path})"
         return super().__repr__()
 
+    def __deepcopy__(self, memo):
+        """
+        Implements support for the standard library copy.deepcopy()
+        function on DT instances.
+        """
+
+        # We need a new DT, obviously. Make a new, empty one.
+        ret = DT(None, (), self._force)
+
+        # Now allocate new Node objects for every node in self, to use
+        # in the new DT. Set their parents to None for now and leave
+        # them without any properties. We will recursively initialize
+        # copies of parents before copies of children next.
+        path2node_copy = {
+            node.path: Node(node.name, None, ret)
+            for node in self.node_iter()
+        }
+
+        # Point each copy of a node to the copy of its parent and set up
+        # copies of each property.
+        #
+        # Share data when possible. For example, Property.value has
+        # type 'bytes', which is immutable. We therefore don't need a
+        # copy and can just point to the original data.
+
+        for node in self.node_iter():
+            node_copy = path2node_copy[node.path]
+
+            parent = node.parent
+            if parent is not None:
+                node_copy.parent = path2node_copy[parent.path]
+
+            prop_name2prop_copy = {
+                prop.name: Property(node_copy, prop.name)
+                for prop in node.props.values()
+            }
+            for prop_name, prop_copy in prop_name2prop_copy.items():
+                prop = node.props[prop_name]
+                prop_copy.value = prop.value
+                prop_copy.labels = prop.labels[:]
+                prop_copy.offset_labels = prop.offset_labels.copy()
+                prop_copy._label_offset_lst = prop._label_offset_lst[:]
+                prop_copy._markers = [marker[:] for marker in prop._markers]
+            node_copy.props = prop_name2prop_copy
+
+            node_copy.nodes = {
+                child_name: path2node_copy[child_node.path]
+                for child_name, child_node in node.nodes.items()
+            }
+
+            node_copy.labels = node.labels[:]
+
+            node_copy._omit_if_no_ref = node._omit_if_no_ref
+            node_copy._is_referenced = node._is_referenced
+
+        # The copied nodes and properties are initialized, so
+        # we can finish initializing the copied DT object now.
+
+        ret._root = path2node_copy['/']
+
+        def copy_node_lookup_table(attr_name):
+            original = getattr(self, attr_name)
+            copy = {
+                key: path2node_copy[original[key].path]
+                for key in original
+            }
+            setattr(ret, attr_name, copy)
+
+        copy_node_lookup_table('alias2node')
+        copy_node_lookup_table('label2node')
+        copy_node_lookup_table('phandle2node')
+
+        ret_label2prop = {}
+        for label, prop in self.label2prop.items():
+            node_copy = path2node_copy[prop.node.path]
+            prop_copy = node_copy.props[prop.name]
+            ret_label2prop[label] = prop_copy
+        ret.label2prop = ret_label2prop
+
+        ret_label2prop_offset = {}
+        for label, prop_offset in self.label2prop_offset.items():
+            prop, offset = prop_offset
+            node_copy = path2node_copy[prop.node.path]
+            prop_copy = node_copy.props[prop.name]
+            ret_label2prop_offset[label] = (prop_copy, offset)
+        ret.label2prop_offset = ret_label2prop_offset
+
+        ret.memreserves = [
+            (set(memreserve[0]), memreserve[1], memreserve[2])
+            for memreserve in self.memreserves
+        ]
+
+        ret.filename = self.filename
+
+        return ret
+
     #
     # Parsing
     #

scripts/dts/python-devicetree/tests/test_dtlib.py

@@ -5,6 +5,7 @@
 import os
 import re
 import tempfile
+from copy import deepcopy
 
 import pytest
 
@@ -2304,3 +2305,114 @@ def test_duplicate_nodes():
 	};
 };
 """)
+
+def test_deepcopy():
+    dt = parse('''
+/dts-v1/;
+
+memreservelabel: /memreserve/ 0xdeadbeef 0x4000;
+
+/ {
+	aliases {
+		foo = &nodelabel;
+	};
+	rootprop_label: rootprop = prop_offset0: <0x12345678 prop_offset4: 0x0>;
+	nodelabel: node@1234 {
+		nodeprop = <3>;
+		subnode {
+			ref-to-node = <&nodelabel>;
+		};
+	};
+};
+''')
+    dt_copy = deepcopy(dt)
+    assert dt_copy.filename == dt.filename
+
+    # dt_copy.root checks:
+    root_copy = dt_copy.root
+    assert root_copy is not dt.root
+    assert root_copy.parent is None
+    assert root_copy.dt is dt_copy
+    assert root_copy.labels == []
+    assert root_copy.labels is not dt.root.labels
+
+    # dt_copy.memreserves checks:
+    assert dt_copy.memreserves == [
+        (set(['memreservelabel']), 0xdeadbeef, 0x4000)
+    ]
+    assert dt_copy.memreserves is not dt.memreserves
+
+    # Miscellaneous dt_copy node and property checks:
+    assert 'rootprop' in root_copy.props
+    rootprop_copy = root_copy.props['rootprop']
+    assert rootprop_copy is not dt.root.props['rootprop']
+    assert rootprop_copy.name == 'rootprop'
+    assert rootprop_copy.value == b'\x12\x34\x56\x78\0\0\0\0'
+    assert rootprop_copy.type == dtlib.Type.NUMS
+    assert rootprop_copy.labels == ['rootprop_label']
+    assert rootprop_copy.labels is not dt.root.props['rootprop'].labels
+    assert rootprop_copy.offset_labels == {
+        'prop_offset0': 0,
+        'prop_offset4': 4,
+    }
+    assert rootprop_copy.offset_labels is not \
+        dt.root.props['rootprop'].offset_labels
+    assert rootprop_copy.node is root_copy
+
+    assert dt_copy.has_node('/node@1234')
+    node_copy = dt_copy.get_node('/node@1234')
+    assert node_copy is not dt.get_node('/node@1234')
+    assert node_copy.labels == ['nodelabel']
+    assert node_copy.labels is not dt.get_node('/node@1234').labels
+    assert node_copy.name == 'node@1234'
+    assert node_copy.unit_addr == '1234'
+    assert node_copy.path == '/node@1234'
+    assert set(node_copy.props.keys()) == set(['nodeprop', 'phandle'])
+    assert node_copy.props is not dt.get_node('/node@1234').props
+    assert node_copy.props['nodeprop'].name == 'nodeprop'
+    assert node_copy.props['nodeprop'].labels == []
+    assert node_copy.props['nodeprop'].offset_labels == {}
+    assert node_copy.props['nodeprop'].node is node_copy
+    assert node_copy.dt is dt_copy
+
+    assert 'subnode' in node_copy.nodes
+    subnode_copy = node_copy.nodes['subnode']
+    assert subnode_copy is not dt.get_node('/node@1234/subnode')
+    assert subnode_copy.parent is node_copy
+
+    # dt_copy.label2prop and .label2prop_offset checks:
+    assert 'rootprop_label' in dt_copy.label2prop
+    assert dt_copy.label2prop['rootprop_label'] is rootprop_copy
+    assert list(dt_copy.label2prop_offset.keys()) == ['prop_offset0',
+                                                      'prop_offset4']
+    assert dt_copy.label2prop_offset['prop_offset4'][0] is rootprop_copy
+    assert dt_copy.label2prop_offset['prop_offset4'][1] == 4
+
+    # dt_copy.foo2node checks:
+    def check_node_lookup_table(attr_name):
+        original = getattr(dt, attr_name)
+        copy = getattr(dt_copy, attr_name)
+        assert original is not copy
+        assert list(original.keys()) == list(copy.keys())
+        assert all([original_node.path == copy_node.path and
+                    original_node is not copy_node
+                    for original_node, copy_node in
+                    zip(original.values(), copy.values())])
+
+    check_node_lookup_table('alias2node')
+    check_node_lookup_table('label2node')
+    check_node_lookup_table('phandle2node')
+
+    assert list(dt_copy.alias2node.keys()) == ['foo']
+    assert dt_copy.alias2node['foo'] is node_copy
+
+    assert list(dt_copy.label2node.keys()) == ['nodelabel']
+    assert dt_copy.label2node['nodelabel'] is node_copy
+
+    assert dt_copy.phandle2node
+    # This is a little awkward because of the way dtlib allocates
+    # phandles.
+    phandle2node_copy_values = set(dt_copy.phandle2node.values())
+    assert node_copy in phandle2node_copy_values
+    for node in dt.node_iter():
+        assert node not in phandle2node_copy_values